bundles/org.simantics.graph.compiler/src/org/simantics/graph/compiler/internal/parsing/Parsing.java

   1 package org.simantics.graph.compiler.internal.parsing;
   2
   3 import gnu.trove.map.hash.THashMap;
   4 import gnu.trove.procedure.TObjectIntProcedure;
   5 import gnu.trove.procedure.TObjectObjectProcedure;
   6
   7 import java.io.ByteArrayOutputStream;
   8 import java.io.IOException;
   9 import java.io.InputStream;
  10 import java.io.PrintStream;
  11 import java.util.ArrayList;
  12 import java.util.Arrays;
  13 import java.util.Collection;
  14 import java.util.Collections;
  15 import java.util.Comparator;
  16 import java.util.concurrent.Callable;
  17 import java.util.concurrent.Future;
  18
  19 import org.antlr.runtime.ANTLRStringStream;
  20 import org.antlr.runtime.CharStream;
  21 import org.antlr.runtime.CommonTokenStream;
  22 import org.antlr.runtime.RecognitionException;
  23 import org.antlr.runtime.TokenStream;
  24 import org.antlr.runtime.tree.Tree;
  25 import org.simantics.graph.compiler.SourceInfo.SourceFile;
  26 import org.simantics.graph.compiler.SourceInfo.Variable;
  27 import org.simantics.graph.compiler.internal.parsing.GraphParser.file_return;
  28 import org.simantics.graph.compiler.internal.parsing.SourceSplitter.SplitPoint;
  29 import org.simantics.graph.compiler.internal.store.VariableStore;
  30 import org.simantics.graph.compiler.internal.translation.GraphTranslator;
  31 import org.simantics.graph.query.Paths;
  32 import org.simantics.graph.store.GraphStore;
  33 import org.simantics.graph.utils.GraphExecutor;
  34 import org.simantics.ltk.ISource;
  35 import org.simantics.ltk.Location;
  36 import org.simantics.ltk.Problem;
  37 import org.simantics.ltk.SourcePart;
  38 import org.simantics.ltk.antlr.ANTLRUtils;
  39
  40 public class Parsing implements Runnable {
  41
  42         Collection<ISource> sources;
  43         Collection<Problem> errors;
  44         GraphStore store;
  45         Paths paths;
  46
  47         public Parsing(Paths paths, Collection<ISource> sources, Collection<Problem> errors,
  48                         GraphStore store) {
  49             this.paths = paths;
  50                 this.sources = sources;
  51                 this.errors = errors;
  52                 this.store = store;
  53         }
  54
  55         private static class ParsingResult {
  56                 ISource source;
  57                 Tree tree;
  58
  59                 public ParsingResult(ISource source, Tree tree) {
  60                         this.source = source;
  61                         this.tree = tree;
  62                 }
  63         }
  64
  65         public static byte[] read(ISource source) throws IOException {
  66             int length = source.length();
  67             if(length >= 0) {
  68                 byte[] buffer = new byte[length];
  69                 InputStream stream = source.open();
  70                 int pos = 0;
  71                 while(pos < buffer.length) {
  72                         int c = stream.read(buffer, pos, buffer.length-pos);
  73                         if(c <= 0)
  74                                 break;
  75                         pos += c;
  76                 }
  77                 stream.close();
  78                 return buffer;
  79             }
  80             else {
  81                 byte[] buffer = new byte[1024];
  82                 InputStream stream = source.open();
  83                 int pos = 0;
  84                 while(true) {
  85                     int count = stream.read(buffer, pos, buffer.length-pos);
  86                     if(count <= 0)
  87                         break;
  88                     pos += count;
  89                     if(pos == buffer.length)
  90                         buffer = Arrays.copyOf(buffer, buffer.length*2);
  91                 }
  92                 stream.close();
  93                 if(pos < buffer.length)
  94                     buffer = Arrays.copyOf(buffer, pos);
  95                 return buffer;
  96             }
  97         }
  98
  99         @Override
 100         public void run() {
 101                 try {
 102                         ArrayList<ISource> orderedSources =
 103                                 new ArrayList<ISource>(sources);
 104                         Collections.sort(orderedSources, new Comparator<ISource>() {
 105                                 @Override
 106                                 public int compare(ISource o1, ISource o2) {
 107                                         return o1.getName().compareTo(o2.getName());
 108                                 }
 109                         });
 110
 111                         ArrayList<Future<ParsingResult>> parsingFutures = new ArrayList<Future<ParsingResult>>();
 112
 113                         //System.out.println("--");
 114                         for(ISource source : orderedSources) {
 115                                 String sourceString = new String(read(source), "UTF-8");
 116                                 ArrayList<SplitPoint> splitPoints = SourceSplitter.split(sourceString, 6400);
 117                                 //System.out.println(splitPoints.size()-1);
 118                                 for(int i=1;i<splitPoints.size();++i) {
 119                                         SplitPoint begin = splitPoints.get(i-1);
 120                                         SplitPoint end = splitPoints.get(i);
 121                                         final String sourcePartString = sourceString.substring(begin.characterId, end.characterId);
 122                                         final SourcePart sourcePart = new SourcePart(source, begin.characterId, begin.lineId);
 123                                         parsingFutures.add(GraphExecutor.EXECUTOR.submit(new Callable<ParsingResult>() {
 124                                                 @Override
 125                                                 public ParsingResult call() throws Exception {
 126                                                         CharStream stream = new ANTLRStringStream(sourcePartString);
 127
 128                                                         GraphLexer lexer = new GraphLexer(stream) {
 129                                                                 @Override
 130                                                                 public void reportError(RecognitionException e) {
 131                                                                         synchronized(errors) {
 132                                                                                 errors.add(ANTLRUtils.error(sourcePart, e, this));
 133                                                                         }
 134                                                                 }
 135                                                         };
 136                                                         TokenStream tokenStream = new CommonTokenStream(lexer);
 137
 138                                                         GraphParser parser = new GraphParser(tokenStream) {
 139                                                                 @Override
 140                                                                 public void reportError(RecognitionException e) {
 141                                                                         synchronized(errors) {
 142                                                                                 errors.add(ANTLRUtils.error(sourcePart, e, this));
 143                                                                         }
 144                                                                 }
 145                                                         };
 146
 147                                                         //long begin = System.nanoTime();
 148                                                         file_return result = parser.file();
 149                                                         //long end = System.nanoTime();
 150
 151                                                         if(!errors.isEmpty())
 152                                                                 return null;
 153
 154                                                         /*double time = (end-begin)*1e-6;
 155                                                         if(time > 200.0) {
 156                                                                 System.out.println(time + " ms, size " + sourcePartString.length());
 157                                                                 System.out.print(sourcePartString);
 158                                                         }*/
 159                                                         return new ParsingResult(sourcePart, (Tree)result.getTree());
 160                                                 }
 161                                         }));
 162                                 }
 163                         }
 164
 165                         /*
 166                          * We want to add graphs to the store in a fixed order and therefore
 167                          * we do not do this part in the same order as the parsing tasks
 168                          * complete.
 169                          */
 170                         THashMap<ISource, GraphTranslator> translators = new THashMap<ISource, GraphTranslator>();
 171                         for(Future<ParsingResult> future : parsingFutures) {
 172                                 ParsingResult result = future.get();
 173                                 synchronized(errors) {
 174                                         if(!errors.isEmpty())
 175                                                 continue;
 176                                 }
 177
 178                                 // Translate AST to statements
 179                                 //long beginTime = System.nanoTime();
 180                                 ISource originalSource = ((SourcePart)result.source).getOriginalSource();
 181                                 GraphTranslator translator = translators.get(originalSource);
 182                                 if(translator == null) {
 183                                         translator = new GraphTranslator(paths, errors, store);
 184                                         translators.put(originalSource, translator);
 185                                 }
 186                                 translator.setSource(result.source);
 187                                 translator.translateGraph(result.tree);
 188                                 /*long endTime = System.nanoTime();
 189                                 reportTime("Trans " + result.source.getName(), beginTime, endTime);
 190                                 */
 191                         }
 192
 193                         /*for(ISource source : orderedSources) {
 194                                 System.out.println(source.getName());
 195                                 GraphTranslator translator = translators.get(source);
 196                                 translator.getVariables().forEachEntry(new TObjectIntProcedure<String>() {
 197                                         @Override
 198                                         public boolean execute(String a, int b) {
 199                                                 System.out.println("    " + a + " " + b);
 200                                                 return true;
 201                                         }
 202                                 });
 203                         }*/
 204
 205                         final VariableStore variableStore = new VariableStore();
 206                         store.addStore(VariableStore.class, variableStore);
 207
 208                         translators.forEachEntry(new TObjectObjectProcedure<ISource, GraphTranslator>() {
 209                                 @Override
 210                                 public boolean execute(ISource a, GraphTranslator b) {
 211                                         final ArrayList<Variable> variables = new ArrayList<Variable>();
 212                                         b.getVariables().forEachEntry(new TObjectIntProcedure<String>() {
 213                                                 @Override
 214                                                 public boolean execute(String a, int b) {
 215                                                         variables.add(new Variable(a, b));
 216                                                         return true;
 217                                                 }
 218                                         });
 219                                         variableStore.addSourceFile(
 220                                                         new SourceFile(a.getName(),
 221                                                                         variables,
 222                                                                         b.getDefinitionPositions()));
 223                                         return true;
 224                                 }
 225                         });
 226                 } catch(Exception e) {
 227                         e.printStackTrace();
 228                         ByteArrayOutputStream stream = new ByteArrayOutputStream();
 229                         e.printStackTrace(new PrintStream(stream));
 230                         String description = "Internal error: " +
 231                                 new String(stream.toByteArray());
 232                         for(ISource source : sources)
 233                                 errors.add(new Problem(new Location(source), description));
 234                 }
 235         }
 236
 237 }