1 package org.simantics.graph.compiler.internal.parsing;
\r
3 import gnu.trove.map.hash.THashMap;
\r
4 import gnu.trove.procedure.TObjectIntProcedure;
\r
5 import gnu.trove.procedure.TObjectObjectProcedure;
\r
7 import java.io.ByteArrayOutputStream;
\r
8 import java.io.IOException;
\r
9 import java.io.InputStream;
\r
10 import java.io.PrintStream;
\r
11 import java.util.ArrayList;
\r
12 import java.util.Arrays;
\r
13 import java.util.Collection;
\r
14 import java.util.Collections;
\r
15 import java.util.Comparator;
\r
16 import java.util.concurrent.Callable;
\r
17 import java.util.concurrent.Future;
\r
19 import org.antlr.runtime.ANTLRStringStream;
\r
20 import org.antlr.runtime.CharStream;
\r
21 import org.antlr.runtime.CommonTokenStream;
\r
22 import org.antlr.runtime.RecognitionException;
\r
23 import org.antlr.runtime.TokenStream;
\r
24 import org.antlr.runtime.tree.Tree;
\r
25 import org.simantics.graph.compiler.SourceInfo.SourceFile;
\r
26 import org.simantics.graph.compiler.SourceInfo.Variable;
\r
27 import org.simantics.graph.compiler.internal.parsing.GraphParser.file_return;
\r
28 import org.simantics.graph.compiler.internal.parsing.SourceSplitter.SplitPoint;
\r
29 import org.simantics.graph.compiler.internal.store.VariableStore;
\r
30 import org.simantics.graph.compiler.internal.translation.GraphTranslator;
\r
31 import org.simantics.graph.query.Paths;
\r
32 import org.simantics.graph.store.GraphStore;
\r
33 import org.simantics.graph.utils.GraphExecutor;
\r
34 import org.simantics.ltk.ISource;
\r
35 import org.simantics.ltk.Location;
\r
36 import org.simantics.ltk.Problem;
\r
37 import org.simantics.ltk.SourcePart;
\r
38 import org.simantics.ltk.antlr.ANTLRUtils;
\r
40 public class Parsing implements Runnable {
\r
42 Collection<ISource> sources;
\r
43 Collection<Problem> errors;
\r
47 public Parsing(Paths paths, Collection<ISource> sources, Collection<Problem> errors,
\r
50 this.sources = sources;
\r
51 this.errors = errors;
\r
55 private static class ParsingResult {
\r
59 public ParsingResult(ISource source, Tree tree) {
\r
60 this.source = source;
\r
65 public static byte[] read(ISource source) throws IOException {
\r
66 int length = source.length();
\r
68 byte[] buffer = new byte[length];
\r
69 InputStream stream = source.open();
\r
71 while(pos < buffer.length) {
\r
72 int c = stream.read(buffer, pos, buffer.length-pos);
\r
81 byte[] buffer = new byte[1024];
\r
82 InputStream stream = source.open();
\r
85 int count = stream.read(buffer, pos, buffer.length-pos);
\r
89 if(pos == buffer.length)
\r
90 buffer = Arrays.copyOf(buffer, buffer.length*2);
\r
93 if(pos < buffer.length)
\r
94 buffer = Arrays.copyOf(buffer, pos);
\r
100 public void run() {
\r
102 ArrayList<ISource> orderedSources =
\r
103 new ArrayList<ISource>(sources);
\r
104 Collections.sort(orderedSources, new Comparator<ISource>() {
\r
106 public int compare(ISource o1, ISource o2) {
\r
107 return o1.getName().compareTo(o2.getName());
\r
111 ArrayList<Future<ParsingResult>> parsingFutures = new ArrayList<Future<ParsingResult>>();
\r
113 //System.out.println("--");
\r
114 for(ISource source : orderedSources) {
\r
115 String sourceString = new String(read(source), "UTF-8");
\r
116 ArrayList<SplitPoint> splitPoints = SourceSplitter.split(sourceString, 6400);
\r
117 //System.out.println(splitPoints.size()-1);
\r
118 for(int i=1;i<splitPoints.size();++i) {
\r
119 SplitPoint begin = splitPoints.get(i-1);
\r
120 SplitPoint end = splitPoints.get(i);
\r
121 final String sourcePartString = sourceString.substring(begin.characterId, end.characterId);
\r
122 final SourcePart sourcePart = new SourcePart(source, begin.characterId, begin.lineId);
\r
123 parsingFutures.add(GraphExecutor.EXECUTOR.submit(new Callable<ParsingResult>() {
\r
125 public ParsingResult call() throws Exception {
\r
126 CharStream stream = new ANTLRStringStream(sourcePartString);
\r
128 GraphLexer lexer = new GraphLexer(stream) {
\r
130 public void reportError(RecognitionException e) {
\r
131 synchronized(errors) {
\r
132 errors.add(ANTLRUtils.error(sourcePart, e, this));
\r
136 TokenStream tokenStream = new CommonTokenStream(lexer);
\r
138 GraphParser parser = new GraphParser(tokenStream) {
\r
140 public void reportError(RecognitionException e) {
\r
141 synchronized(errors) {
\r
142 errors.add(ANTLRUtils.error(sourcePart, e, this));
\r
147 //long begin = System.nanoTime();
\r
148 file_return result = parser.file();
\r
149 //long end = System.nanoTime();
\r
151 if(!errors.isEmpty())
\r
154 /*double time = (end-begin)*1e-6;
\r
156 System.out.println(time + " ms, size " + sourcePartString.length());
\r
157 System.out.print(sourcePartString);
\r
159 return new ParsingResult(sourcePart, (Tree)result.getTree());
\r
166 * We want to add graphs to the store in a fixed order and therefore
\r
167 * we do not do this part in the same order as the parsing tasks
\r
170 THashMap<ISource, GraphTranslator> translators = new THashMap<ISource, GraphTranslator>();
\r
171 for(Future<ParsingResult> future : parsingFutures) {
\r
172 ParsingResult result = future.get();
\r
173 synchronized(errors) {
\r
174 if(!errors.isEmpty())
\r
178 // Translate AST to statements
\r
179 //long beginTime = System.nanoTime();
\r
180 ISource originalSource = ((SourcePart)result.source).getOriginalSource();
\r
181 GraphTranslator translator = translators.get(originalSource);
\r
182 if(translator == null) {
\r
183 translator = new GraphTranslator(paths, errors, store);
\r
184 translators.put(originalSource, translator);
\r
186 translator.setSource(result.source);
\r
187 translator.translateGraph(result.tree);
\r
188 /*long endTime = System.nanoTime();
\r
189 reportTime("Trans " + result.source.getName(), beginTime, endTime);
\r
193 /*for(ISource source : orderedSources) {
\r
194 System.out.println(source.getName());
\r
195 GraphTranslator translator = translators.get(source);
\r
196 translator.getVariables().forEachEntry(new TObjectIntProcedure<String>() {
\r
198 public boolean execute(String a, int b) {
\r
199 System.out.println(" " + a + " " + b);
\r
205 final VariableStore variableStore = new VariableStore();
\r
206 store.addStore(VariableStore.class, variableStore);
\r
208 translators.forEachEntry(new TObjectObjectProcedure<ISource, GraphTranslator>() {
\r
210 public boolean execute(ISource a, GraphTranslator b) {
\r
211 final ArrayList<Variable> variables = new ArrayList<Variable>();
\r
212 b.getVariables().forEachEntry(new TObjectIntProcedure<String>() {
\r
214 public boolean execute(String a, int b) {
\r
215 variables.add(new Variable(a, b));
\r
219 variableStore.addSourceFile(
\r
220 new SourceFile(a.getName(),
\r
222 b.getDefinitionPositions()));
\r
226 } catch(Exception e) {
\r
227 e.printStackTrace();
\r
228 ByteArrayOutputStream stream = new ByteArrayOutputStream();
\r
229 e.printStackTrace(new PrintStream(stream));
\r
230 String description = "Internal error: " +
\r
231 new String(stream.toByteArray());
\r
232 for(ISource source : sources)
\r
233 errors.add(new Problem(new Location(source), description));
\r