bundles/org.simantics.graph.compiler/src/org/simantics/graph/compiler/internal/parsing/Parsing.java

   1 package org.simantics.graph.compiler.internal.parsing;\r
   2 \r
   3 import gnu.trove.map.hash.THashMap;\r
   4 import gnu.trove.procedure.TObjectIntProcedure;\r
   5 import gnu.trove.procedure.TObjectObjectProcedure;\r
   6 \r
   7 import java.io.ByteArrayOutputStream;\r
   8 import java.io.IOException;\r
   9 import java.io.InputStream;\r
  10 import java.io.PrintStream;\r
  11 import java.util.ArrayList;\r
  12 import java.util.Arrays;\r
  13 import java.util.Collection;\r
  14 import java.util.Collections;\r
  15 import java.util.Comparator;\r
  16 import java.util.concurrent.Callable;\r
  17 import java.util.concurrent.Future;\r
  18 \r
  19 import org.antlr.runtime.ANTLRStringStream;\r
  20 import org.antlr.runtime.CharStream;\r
  21 import org.antlr.runtime.CommonTokenStream;\r
  22 import org.antlr.runtime.RecognitionException;\r
  23 import org.antlr.runtime.TokenStream;\r
  24 import org.antlr.runtime.tree.Tree;\r
  25 import org.simantics.graph.compiler.SourceInfo.SourceFile;\r
  26 import org.simantics.graph.compiler.SourceInfo.Variable;\r
  27 import org.simantics.graph.compiler.internal.parsing.GraphParser.file_return;\r
  28 import org.simantics.graph.compiler.internal.parsing.SourceSplitter.SplitPoint;\r
  29 import org.simantics.graph.compiler.internal.store.VariableStore;\r
  30 import org.simantics.graph.compiler.internal.translation.GraphTranslator;\r
  31 import org.simantics.graph.query.Paths;\r
  32 import org.simantics.graph.store.GraphStore;\r
  33 import org.simantics.graph.utils.GraphExecutor;\r
  34 import org.simantics.ltk.ISource;\r
  35 import org.simantics.ltk.Location;\r
  36 import org.simantics.ltk.Problem;\r
  37 import org.simantics.ltk.SourcePart;\r
  38 import org.simantics.ltk.antlr.ANTLRUtils;\r
  39 \r
  40 public class Parsing implements Runnable {\r
  41 \r
  42         Collection<ISource> sources;\r
  43         Collection<Problem> errors;\r
  44         GraphStore store;\r
  45         Paths paths;\r
  46         \r
  47         public Parsing(Paths paths, Collection<ISource> sources, Collection<Problem> errors,\r
  48                         GraphStore store) {\r
  49             this.paths = paths;\r
  50                 this.sources = sources;\r
  51                 this.errors = errors;\r
  52                 this.store = store;\r
  53         }\r
  54 \r
  55         private static class ParsingResult {\r
  56                 ISource source;\r
  57                 Tree tree;\r
  58                 \r
  59                 public ParsingResult(ISource source, Tree tree) {\r
  60                         this.source = source;\r
  61                         this.tree = tree;\r
  62                 }\r
  63         }\r
  64         \r
  65         public static byte[] read(ISource source) throws IOException {\r
  66             int length = source.length();\r
  67             if(length >= 0) {\r
  68                 byte[] buffer = new byte[length];\r
  69                 InputStream stream = source.open();\r
  70                 int pos = 0;\r
  71                 while(pos < buffer.length) {\r
  72                         int c = stream.read(buffer, pos, buffer.length-pos);\r
  73                         if(c <= 0)\r
  74                                 break;\r
  75                         pos += c;\r
  76                 }               \r
  77                 stream.close();         \r
  78                 return buffer;\r
  79             }\r
  80             else {\r
  81                 byte[] buffer = new byte[1024];\r
  82                 InputStream stream = source.open();\r
  83                 int pos = 0;\r
  84                 while(true) {\r
  85                     int count = stream.read(buffer, pos, buffer.length-pos);\r
  86                     if(count <= 0)\r
  87                         break;\r
  88                     pos += count;\r
  89                     if(pos == buffer.length)\r
  90                         buffer = Arrays.copyOf(buffer, buffer.length*2);\r
  91                 }\r
  92                 stream.close();\r
  93                 if(pos < buffer.length)\r
  94                     buffer = Arrays.copyOf(buffer, pos);\r
  95                 return buffer;\r
  96             }\r
  97         }\r
  98         \r
  99         @Override\r
 100         public void run() {\r
 101                 try {\r
 102                         ArrayList<ISource> orderedSources = \r
 103                                 new ArrayList<ISource>(sources);\r
 104                         Collections.sort(orderedSources, new Comparator<ISource>() {\r
 105                                 @Override\r
 106                                 public int compare(ISource o1, ISource o2) {\r
 107                                         return o1.getName().compareTo(o2.getName());\r
 108                                 }\r
 109                         });\r
 110                         \r
 111                         ArrayList<Future<ParsingResult>> parsingFutures = new ArrayList<Future<ParsingResult>>(); \r
 112         \r
 113                         //System.out.println("--");\r
 114                         for(ISource source : orderedSources) {\r
 115                                 String sourceString = new String(read(source), "UTF-8");                                \r
 116                                 ArrayList<SplitPoint> splitPoints = SourceSplitter.split(sourceString, 6400);                   \r
 117                                 //System.out.println(splitPoints.size()-1);\r
 118                                 for(int i=1;i<splitPoints.size();++i) {\r
 119                                         SplitPoint begin = splitPoints.get(i-1);\r
 120                                         SplitPoint end = splitPoints.get(i);\r
 121                                         final String sourcePartString = sourceString.substring(begin.characterId, end.characterId);\r
 122                                         final SourcePart sourcePart = new SourcePart(source, begin.characterId, begin.lineId);\r
 123                                         parsingFutures.add(GraphExecutor.EXECUTOR.submit(new Callable<ParsingResult>() {\r
 124                                                 @Override\r
 125                                                 public ParsingResult call() throws Exception {                                                  \r
 126                                                         CharStream stream = new ANTLRStringStream(sourcePartString);\r
 127                                                         \r
 128                                                         GraphLexer lexer = new GraphLexer(stream) {\r
 129                                                                 @Override\r
 130                                                                 public void reportError(RecognitionException e) {\r
 131                                                                         synchronized(errors) {\r
 132                                                                                 errors.add(ANTLRUtils.error(sourcePart, e, this));\r
 133                                                                         }\r
 134                                                                 }\r
 135                                                         };\r
 136                                                         TokenStream tokenStream = new CommonTokenStream(lexer);         \r
 137         \r
 138                                                         GraphParser parser = new GraphParser(tokenStream) {\r
 139                                                                 @Override\r
 140                                                                 public void reportError(RecognitionException e) {\r
 141                                                                         synchronized(errors) {\r
 142                                                                                 errors.add(ANTLRUtils.error(sourcePart, e, this));\r
 143                                                                         }\r
 144                                                                 }\r
 145                                                         };      \r
 146         \r
 147                                                         //long begin = System.nanoTime();\r
 148                                                         file_return result = parser.file();\r
 149                                                         //long end = System.nanoTime();\r
 150                                                         \r
 151                                                         if(!errors.isEmpty())\r
 152                                                                 return null;            \r
 153                                                         \r
 154                                                         /*double time = (end-begin)*1e-6;                                                       \r
 155                                                         if(time > 200.0) {\r
 156                                                                 System.out.println(time + " ms, size " + sourcePartString.length());\r
 157                                                                 System.out.print(sourcePartString);\r
 158                                                         }*/\r
 159                                                         return new ParsingResult(sourcePart, (Tree)result.getTree());\r
 160                                                 }\r
 161                                         }));\r
 162                                 }\r
 163                         }\r
 164                         \r
 165                         /*\r
 166                          * We want to add graphs to the store in a fixed order and therefore\r
 167                          * we do not do this part in the same order as the parsing tasks\r
 168                          * complete.\r
 169                          */\r
 170                         THashMap<ISource, GraphTranslator> translators = new THashMap<ISource, GraphTranslator>(); \r
 171                         for(Future<ParsingResult> future : parsingFutures) {                            \r
 172                                 ParsingResult result = future.get();\r
 173                                 synchronized(errors) {\r
 174                                         if(!errors.isEmpty())\r
 175                                                 continue;\r
 176                                 }\r
 177                                         \r
 178                                 // Translate AST to statements\r
 179                                 //long beginTime = System.nanoTime();\r
 180                                 ISource originalSource = ((SourcePart)result.source).getOriginalSource();\r
 181                                 GraphTranslator translator = translators.get(originalSource);\r
 182                                 if(translator == null) {\r
 183                                         translator = new GraphTranslator(paths, errors, store);\r
 184                                         translators.put(originalSource, translator);\r
 185                                 }\r
 186                                 translator.setSource(result.source);\r
 187                                 translator.translateGraph(result.tree);\r
 188                                 /*long endTime = System.nanoTime();\r
 189                                 reportTime("Trans " + result.source.getName(), beginTime, endTime);\r
 190                                 */\r
 191                         }\r
 192                         \r
 193                         /*for(ISource source : orderedSources) {\r
 194                                 System.out.println(source.getName());\r
 195                                 GraphTranslator translator = translators.get(source);\r
 196                                 translator.getVariables().forEachEntry(new TObjectIntProcedure<String>() {                                      \r
 197                                         @Override\r
 198                                         public boolean execute(String a, int b) {\r
 199                                                 System.out.println("    " + a + " " + b);\r
 200                                                 return true;\r
 201                                         }\r
 202                                 });\r
 203                         }*/\r
 204                         \r
 205                         final VariableStore variableStore = new VariableStore();\r
 206                         store.addStore(VariableStore.class, variableStore);\r
 207                         \r
 208                         translators.forEachEntry(new TObjectObjectProcedure<ISource, GraphTranslator>() {\r
 209                                 @Override\r
 210                                 public boolean execute(ISource a, GraphTranslator b) {\r
 211                                         final ArrayList<Variable> variables = new ArrayList<Variable>();\r
 212                                         b.getVariables().forEachEntry(new TObjectIntProcedure<String>() {\r
 213                                                 @Override\r
 214                                                 public boolean execute(String a, int b) {\r
 215                                                         variables.add(new Variable(a, b));\r
 216                                                         return true;\r
 217                                                 }\r
 218                                         });\r
 219                                         variableStore.addSourceFile(\r
 220                                                         new SourceFile(a.getName(), \r
 221                                                                         variables, \r
 222                                                                         b.getDefinitionPositions()));\r
 223                                         return true;\r
 224                                 }\r
 225                         });\r
 226                 } catch(Exception e) {\r
 227                         e.printStackTrace();\r
 228                         ByteArrayOutputStream stream = new ByteArrayOutputStream();\r
 229                         e.printStackTrace(new PrintStream(stream));\r
 230                         String description = "Internal error: " +\r
 231                                 new String(stream.toByteArray());\r
 232                         for(ISource source : sources)           \r
 233                                 errors.add(new Problem(new Location(source), description));\r
 234                 }\r
 235         }\r
 236 \r
 237 }\r