-package org.simantics.graph.compiler.internal.parsing;\r
-\r
-import gnu.trove.map.hash.THashMap;\r
-import gnu.trove.procedure.TObjectIntProcedure;\r
-import gnu.trove.procedure.TObjectObjectProcedure;\r
-\r
-import java.io.ByteArrayOutputStream;\r
-import java.io.IOException;\r
-import java.io.InputStream;\r
-import java.io.PrintStream;\r
-import java.util.ArrayList;\r
-import java.util.Arrays;\r
-import java.util.Collection;\r
-import java.util.Collections;\r
-import java.util.Comparator;\r
-import java.util.concurrent.Callable;\r
-import java.util.concurrent.Future;\r
-\r
-import org.antlr.runtime.ANTLRStringStream;\r
-import org.antlr.runtime.CharStream;\r
-import org.antlr.runtime.CommonTokenStream;\r
-import org.antlr.runtime.RecognitionException;\r
-import org.antlr.runtime.TokenStream;\r
-import org.antlr.runtime.tree.Tree;\r
-import org.simantics.graph.compiler.SourceInfo.SourceFile;\r
-import org.simantics.graph.compiler.SourceInfo.Variable;\r
-import org.simantics.graph.compiler.internal.parsing.GraphParser.file_return;\r
-import org.simantics.graph.compiler.internal.parsing.SourceSplitter.SplitPoint;\r
-import org.simantics.graph.compiler.internal.store.VariableStore;\r
-import org.simantics.graph.compiler.internal.translation.GraphTranslator;\r
-import org.simantics.graph.query.Paths;\r
-import org.simantics.graph.store.GraphStore;\r
-import org.simantics.graph.utils.GraphExecutor;\r
-import org.simantics.ltk.ISource;\r
-import org.simantics.ltk.Location;\r
-import org.simantics.ltk.Problem;\r
-import org.simantics.ltk.SourcePart;\r
-import org.simantics.ltk.antlr.ANTLRUtils;\r
-\r
-public class Parsing implements Runnable {\r
-\r
- Collection<ISource> sources;\r
- Collection<Problem> errors;\r
- GraphStore store;\r
- Paths paths;\r
- \r
- public Parsing(Paths paths, Collection<ISource> sources, Collection<Problem> errors,\r
- GraphStore store) {\r
- this.paths = paths;\r
- this.sources = sources;\r
- this.errors = errors;\r
- this.store = store;\r
- }\r
-\r
- private static class ParsingResult {\r
- ISource source;\r
- Tree tree;\r
- \r
- public ParsingResult(ISource source, Tree tree) {\r
- this.source = source;\r
- this.tree = tree;\r
- }\r
- }\r
- \r
- public static byte[] read(ISource source) throws IOException {\r
- int length = source.length();\r
- if(length >= 0) {\r
- byte[] buffer = new byte[length];\r
- InputStream stream = source.open();\r
- int pos = 0;\r
- while(pos < buffer.length) {\r
- int c = stream.read(buffer, pos, buffer.length-pos);\r
- if(c <= 0)\r
- break;\r
- pos += c;\r
- } \r
- stream.close(); \r
- return buffer;\r
- }\r
- else {\r
- byte[] buffer = new byte[1024];\r
- InputStream stream = source.open();\r
- int pos = 0;\r
- while(true) {\r
- int count = stream.read(buffer, pos, buffer.length-pos);\r
- if(count <= 0)\r
- break;\r
- pos += count;\r
- if(pos == buffer.length)\r
- buffer = Arrays.copyOf(buffer, buffer.length*2);\r
- }\r
- stream.close();\r
- if(pos < buffer.length)\r
- buffer = Arrays.copyOf(buffer, pos);\r
- return buffer;\r
- }\r
- }\r
- \r
- @Override\r
- public void run() {\r
- try {\r
- ArrayList<ISource> orderedSources = \r
- new ArrayList<ISource>(sources);\r
- Collections.sort(orderedSources, new Comparator<ISource>() {\r
- @Override\r
- public int compare(ISource o1, ISource o2) {\r
- return o1.getName().compareTo(o2.getName());\r
- }\r
- });\r
- \r
- ArrayList<Future<ParsingResult>> parsingFutures = new ArrayList<Future<ParsingResult>>(); \r
- \r
- //System.out.println("--");\r
- for(ISource source : orderedSources) {\r
- String sourceString = new String(read(source), "UTF-8"); \r
- ArrayList<SplitPoint> splitPoints = SourceSplitter.split(sourceString, 6400); \r
- //System.out.println(splitPoints.size()-1);\r
- for(int i=1;i<splitPoints.size();++i) {\r
- SplitPoint begin = splitPoints.get(i-1);\r
- SplitPoint end = splitPoints.get(i);\r
- final String sourcePartString = sourceString.substring(begin.characterId, end.characterId);\r
- final SourcePart sourcePart = new SourcePart(source, begin.characterId, begin.lineId);\r
- parsingFutures.add(GraphExecutor.EXECUTOR.submit(new Callable<ParsingResult>() {\r
- @Override\r
- public ParsingResult call() throws Exception { \r
- CharStream stream = new ANTLRStringStream(sourcePartString);\r
- \r
- GraphLexer lexer = new GraphLexer(stream) {\r
- @Override\r
- public void reportError(RecognitionException e) {\r
- synchronized(errors) {\r
- errors.add(ANTLRUtils.error(sourcePart, e, this));\r
- }\r
- }\r
- };\r
- TokenStream tokenStream = new CommonTokenStream(lexer); \r
- \r
- GraphParser parser = new GraphParser(tokenStream) {\r
- @Override\r
- public void reportError(RecognitionException e) {\r
- synchronized(errors) {\r
- errors.add(ANTLRUtils.error(sourcePart, e, this));\r
- }\r
- }\r
- }; \r
- \r
- //long begin = System.nanoTime();\r
- file_return result = parser.file();\r
- //long end = System.nanoTime();\r
- \r
- if(!errors.isEmpty())\r
- return null; \r
- \r
- /*double time = (end-begin)*1e-6; \r
- if(time > 200.0) {\r
- System.out.println(time + " ms, size " + sourcePartString.length());\r
- System.out.print(sourcePartString);\r
- }*/\r
- return new ParsingResult(sourcePart, (Tree)result.getTree());\r
- }\r
- }));\r
- }\r
- }\r
- \r
- /*\r
- * We want to add graphs to the store in a fixed order and therefore\r
- * we do not do this part in the same order as the parsing tasks\r
- * complete.\r
- */\r
- THashMap<ISource, GraphTranslator> translators = new THashMap<ISource, GraphTranslator>(); \r
- for(Future<ParsingResult> future : parsingFutures) { \r
- ParsingResult result = future.get();\r
- synchronized(errors) {\r
- if(!errors.isEmpty())\r
- continue;\r
- }\r
- \r
- // Translate AST to statements\r
- //long beginTime = System.nanoTime();\r
- ISource originalSource = ((SourcePart)result.source).getOriginalSource();\r
- GraphTranslator translator = translators.get(originalSource);\r
- if(translator == null) {\r
- translator = new GraphTranslator(paths, errors, store);\r
- translators.put(originalSource, translator);\r
- }\r
- translator.setSource(result.source);\r
- translator.translateGraph(result.tree);\r
- /*long endTime = System.nanoTime();\r
- reportTime("Trans " + result.source.getName(), beginTime, endTime);\r
- */\r
- }\r
- \r
- /*for(ISource source : orderedSources) {\r
- System.out.println(source.getName());\r
- GraphTranslator translator = translators.get(source);\r
- translator.getVariables().forEachEntry(new TObjectIntProcedure<String>() { \r
- @Override\r
- public boolean execute(String a, int b) {\r
- System.out.println(" " + a + " " + b);\r
- return true;\r
- }\r
- });\r
- }*/\r
- \r
- final VariableStore variableStore = new VariableStore();\r
- store.addStore(VariableStore.class, variableStore);\r
- \r
- translators.forEachEntry(new TObjectObjectProcedure<ISource, GraphTranslator>() {\r
- @Override\r
- public boolean execute(ISource a, GraphTranslator b) {\r
- final ArrayList<Variable> variables = new ArrayList<Variable>();\r
- b.getVariables().forEachEntry(new TObjectIntProcedure<String>() {\r
- @Override\r
- public boolean execute(String a, int b) {\r
- variables.add(new Variable(a, b));\r
- return true;\r
- }\r
- });\r
- variableStore.addSourceFile(\r
- new SourceFile(a.getName(), \r
- variables, \r
- b.getDefinitionPositions()));\r
- return true;\r
- }\r
- });\r
- } catch(Exception e) {\r
- e.printStackTrace();\r
- ByteArrayOutputStream stream = new ByteArrayOutputStream();\r
- e.printStackTrace(new PrintStream(stream));\r
- String description = "Internal error: " +\r
- new String(stream.toByteArray());\r
- for(ISource source : sources) \r
- errors.add(new Problem(new Location(source), description));\r
- }\r
- }\r
-\r
-}\r
+package org.simantics.graph.compiler.internal.parsing;
+
+import gnu.trove.map.hash.THashMap;
+import gnu.trove.procedure.TObjectIntProcedure;
+import gnu.trove.procedure.TObjectObjectProcedure;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Future;
+
+import org.antlr.runtime.ANTLRStringStream;
+import org.antlr.runtime.CharStream;
+import org.antlr.runtime.CommonTokenStream;
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.TokenStream;
+import org.antlr.runtime.tree.Tree;
+import org.simantics.graph.compiler.SourceInfo.SourceFile;
+import org.simantics.graph.compiler.SourceInfo.Variable;
+import org.simantics.graph.compiler.internal.parsing.GraphParser.file_return;
+import org.simantics.graph.compiler.internal.parsing.SourceSplitter.SplitPoint;
+import org.simantics.graph.compiler.internal.store.VariableStore;
+import org.simantics.graph.compiler.internal.translation.GraphTranslator;
+import org.simantics.graph.query.Paths;
+import org.simantics.graph.store.GraphStore;
+import org.simantics.graph.utils.GraphExecutor;
+import org.simantics.ltk.ISource;
+import org.simantics.ltk.Location;
+import org.simantics.ltk.Problem;
+import org.simantics.ltk.SourcePart;
+import org.simantics.ltk.antlr.ANTLRUtils;
+
+public class Parsing implements Runnable {
+
+ Collection<ISource> sources;
+ Collection<Problem> errors;
+ GraphStore store;
+ Paths paths;
+
+ public Parsing(Paths paths, Collection<ISource> sources, Collection<Problem> errors,
+ GraphStore store) {
+ this.paths = paths;
+ this.sources = sources;
+ this.errors = errors;
+ this.store = store;
+ }
+
+ private static class ParsingResult {
+ ISource source;
+ Tree tree;
+
+ public ParsingResult(ISource source, Tree tree) {
+ this.source = source;
+ this.tree = tree;
+ }
+ }
+
+ public static byte[] read(ISource source) throws IOException {
+ int length = source.length();
+ if(length >= 0) {
+ byte[] buffer = new byte[length];
+ InputStream stream = source.open();
+ int pos = 0;
+ while(pos < buffer.length) {
+ int c = stream.read(buffer, pos, buffer.length-pos);
+ if(c <= 0)
+ break;
+ pos += c;
+ }
+ stream.close();
+ return buffer;
+ }
+ else {
+ byte[] buffer = new byte[1024];
+ InputStream stream = source.open();
+ int pos = 0;
+ while(true) {
+ int count = stream.read(buffer, pos, buffer.length-pos);
+ if(count <= 0)
+ break;
+ pos += count;
+ if(pos == buffer.length)
+ buffer = Arrays.copyOf(buffer, buffer.length*2);
+ }
+ stream.close();
+ if(pos < buffer.length)
+ buffer = Arrays.copyOf(buffer, pos);
+ return buffer;
+ }
+ }
+
+ @Override
+ public void run() {
+ try {
+ ArrayList<ISource> orderedSources =
+ new ArrayList<ISource>(sources);
+ Collections.sort(orderedSources, new Comparator<ISource>() {
+ @Override
+ public int compare(ISource o1, ISource o2) {
+ return o1.getName().compareTo(o2.getName());
+ }
+ });
+
+ ArrayList<Future<ParsingResult>> parsingFutures = new ArrayList<Future<ParsingResult>>();
+
+ //System.out.println("--");
+ for(ISource source : orderedSources) {
+ String sourceString = new String(read(source), "UTF-8");
+ ArrayList<SplitPoint> splitPoints = SourceSplitter.split(sourceString, 6400);
+ //System.out.println(splitPoints.size()-1);
+ for(int i=1;i<splitPoints.size();++i) {
+ SplitPoint begin = splitPoints.get(i-1);
+ SplitPoint end = splitPoints.get(i);
+ final String sourcePartString = sourceString.substring(begin.characterId, end.characterId);
+ final SourcePart sourcePart = new SourcePart(source, begin.characterId, begin.lineId);
+ parsingFutures.add(GraphExecutor.EXECUTOR.submit(new Callable<ParsingResult>() {
+ @Override
+ public ParsingResult call() throws Exception {
+ CharStream stream = new ANTLRStringStream(sourcePartString);
+
+ GraphLexer lexer = new GraphLexer(stream) {
+ @Override
+ public void reportError(RecognitionException e) {
+ synchronized(errors) {
+ errors.add(ANTLRUtils.error(sourcePart, e, this));
+ }
+ }
+ };
+ TokenStream tokenStream = new CommonTokenStream(lexer);
+
+ GraphParser parser = new GraphParser(tokenStream) {
+ @Override
+ public void reportError(RecognitionException e) {
+ synchronized(errors) {
+ errors.add(ANTLRUtils.error(sourcePart, e, this));
+ }
+ }
+ };
+
+ //long begin = System.nanoTime();
+ file_return result = parser.file();
+ //long end = System.nanoTime();
+
+ if(!errors.isEmpty())
+ return null;
+
+ /*double time = (end-begin)*1e-6;
+ if(time > 200.0) {
+ System.out.println(time + " ms, size " + sourcePartString.length());
+ System.out.print(sourcePartString);
+ }*/
+ return new ParsingResult(sourcePart, (Tree)result.getTree());
+ }
+ }));
+ }
+ }
+
+ /*
+ * We want to add graphs to the store in a fixed order and therefore
+ * we do not do this part in the same order as the parsing tasks
+ * complete.
+ */
+ THashMap<ISource, GraphTranslator> translators = new THashMap<ISource, GraphTranslator>();
+ for(Future<ParsingResult> future : parsingFutures) {
+ ParsingResult result = future.get();
+ synchronized(errors) {
+ if(!errors.isEmpty())
+ continue;
+ }
+
+ // Translate AST to statements
+ //long beginTime = System.nanoTime();
+ ISource originalSource = ((SourcePart)result.source).getOriginalSource();
+ GraphTranslator translator = translators.get(originalSource);
+ if(translator == null) {
+ translator = new GraphTranslator(paths, errors, store);
+ translators.put(originalSource, translator);
+ }
+ translator.setSource(result.source);
+ translator.translateGraph(result.tree);
+ /*long endTime = System.nanoTime();
+ reportTime("Trans " + result.source.getName(), beginTime, endTime);
+ */
+ }
+
+ /*for(ISource source : orderedSources) {
+ System.out.println(source.getName());
+ GraphTranslator translator = translators.get(source);
+ translator.getVariables().forEachEntry(new TObjectIntProcedure<String>() {
+ @Override
+ public boolean execute(String a, int b) {
+ System.out.println(" " + a + " " + b);
+ return true;
+ }
+ });
+ }*/
+
+ final VariableStore variableStore = new VariableStore();
+ store.addStore(VariableStore.class, variableStore);
+
+ translators.forEachEntry(new TObjectObjectProcedure<ISource, GraphTranslator>() {
+ @Override
+ public boolean execute(ISource a, GraphTranslator b) {
+ final ArrayList<Variable> variables = new ArrayList<Variable>();
+ b.getVariables().forEachEntry(new TObjectIntProcedure<String>() {
+ @Override
+ public boolean execute(String a, int b) {
+ variables.add(new Variable(a, b));
+ return true;
+ }
+ });
+ variableStore.addSourceFile(
+ new SourceFile(a.getName(),
+ variables,
+ b.getDefinitionPositions()));
+ return true;
+ }
+ });
+ } catch(Exception e) {
+ e.printStackTrace();
+ ByteArrayOutputStream stream = new ByteArrayOutputStream();
+ e.printStackTrace(new PrintStream(stream));
+ String description = "Internal error: " +
+ new String(stream.toByteArray());
+ for(ISource source : sources)
+ errors.add(new Problem(new Location(source), description));
+ }
+ }
+
+}
Code Review / simantics / platform.git / blobdiff