1 package org.simantics.graph.compiler.internal.parsing;
3 import gnu.trove.map.hash.THashMap;
4 import gnu.trove.procedure.TObjectIntProcedure;
5 import gnu.trove.procedure.TObjectObjectProcedure;
7 import java.io.ByteArrayOutputStream;
8 import java.io.IOException;
9 import java.io.InputStream;
10 import java.io.PrintStream;
11 import java.util.ArrayList;
12 import java.util.Arrays;
13 import java.util.Collection;
14 import java.util.Collections;
15 import java.util.Comparator;
16 import java.util.concurrent.Callable;
17 import java.util.concurrent.Future;
19 import org.antlr.runtime.ANTLRStringStream;
20 import org.antlr.runtime.CharStream;
21 import org.antlr.runtime.CommonTokenStream;
22 import org.antlr.runtime.RecognitionException;
23 import org.antlr.runtime.TokenStream;
24 import org.antlr.runtime.tree.Tree;
25 import org.simantics.graph.compiler.SourceInfo.SourceFile;
26 import org.simantics.graph.compiler.SourceInfo.Variable;
27 import org.simantics.graph.compiler.internal.parsing.GraphParser.file_return;
28 import org.simantics.graph.compiler.internal.parsing.SourceSplitter.SplitPoint;
29 import org.simantics.graph.compiler.internal.store.VariableStore;
30 import org.simantics.graph.compiler.internal.translation.GraphTranslator;
31 import org.simantics.graph.query.Paths;
32 import org.simantics.graph.store.GraphStore;
33 import org.simantics.graph.utils.GraphExecutor;
34 import org.simantics.ltk.ISource;
35 import org.simantics.ltk.Location;
36 import org.simantics.ltk.Problem;
37 import org.simantics.ltk.SourcePart;
38 import org.simantics.ltk.antlr.ANTLRUtils;
40 public class Parsing implements Runnable {
42 Collection<ISource> sources;
43 Collection<Problem> errors;
47 public Parsing(Paths paths, Collection<ISource> sources, Collection<Problem> errors,
50 this.sources = sources;
55 private static class ParsingResult {
59 public ParsingResult(ISource source, Tree tree) {
65 public static byte[] read(ISource source) throws IOException {
66 int length = source.length();
68 byte[] buffer = new byte[length];
69 InputStream stream = source.open();
71 while(pos < buffer.length) {
72 int c = stream.read(buffer, pos, buffer.length-pos);
81 byte[] buffer = new byte[1024];
82 InputStream stream = source.open();
85 int count = stream.read(buffer, pos, buffer.length-pos);
89 if(pos == buffer.length)
90 buffer = Arrays.copyOf(buffer, buffer.length*2);
93 if(pos < buffer.length)
94 buffer = Arrays.copyOf(buffer, pos);
102 ArrayList<ISource> orderedSources =
103 new ArrayList<ISource>(sources);
104 Collections.sort(orderedSources, new Comparator<ISource>() {
106 public int compare(ISource o1, ISource o2) {
107 return o1.getName().compareTo(o2.getName());
111 ArrayList<Future<ParsingResult>> parsingFutures = new ArrayList<Future<ParsingResult>>();
113 //System.out.println("--");
114 for(ISource source : orderedSources) {
115 String sourceString = new String(read(source), "UTF-8");
116 ArrayList<SplitPoint> splitPoints = SourceSplitter.split(sourceString, 6400);
117 //System.out.println(splitPoints.size()-1);
118 for(int i=1;i<splitPoints.size();++i) {
119 SplitPoint begin = splitPoints.get(i-1);
120 SplitPoint end = splitPoints.get(i);
121 final String sourcePartString = sourceString.substring(begin.characterId, end.characterId);
122 final SourcePart sourcePart = new SourcePart(source, begin.characterId, begin.lineId);
123 parsingFutures.add(GraphExecutor.EXECUTOR.submit(new Callable<ParsingResult>() {
125 public ParsingResult call() throws Exception {
126 CharStream stream = new ANTLRStringStream(sourcePartString);
128 GraphLexer lexer = new GraphLexer(stream) {
130 public void reportError(RecognitionException e) {
131 synchronized(errors) {
132 errors.add(ANTLRUtils.error(sourcePart, e, this));
136 TokenStream tokenStream = new CommonTokenStream(lexer);
138 GraphParser parser = new GraphParser(tokenStream) {
140 public void reportError(RecognitionException e) {
141 synchronized(errors) {
142 errors.add(ANTLRUtils.error(sourcePart, e, this));
147 //long begin = System.nanoTime();
148 file_return result = parser.file();
149 //long end = System.nanoTime();
151 if(!errors.isEmpty())
154 /*double time = (end-begin)*1e-6;
156 System.out.println(time + " ms, size " + sourcePartString.length());
157 System.out.print(sourcePartString);
159 return new ParsingResult(sourcePart, (Tree)result.getTree());
166 * We want to add graphs to the store in a fixed order and therefore
167 * we do not do this part in the same order as the parsing tasks
170 THashMap<ISource, GraphTranslator> translators = new THashMap<ISource, GraphTranslator>();
171 for(Future<ParsingResult> future : parsingFutures) {
172 ParsingResult result = future.get();
173 synchronized(errors) {
174 if(!errors.isEmpty())
178 // Translate AST to statements
179 //long beginTime = System.nanoTime();
180 ISource originalSource = ((SourcePart)result.source).getOriginalSource();
181 GraphTranslator translator = translators.get(originalSource);
182 if(translator == null) {
183 translator = new GraphTranslator(paths, errors, store);
184 translators.put(originalSource, translator);
186 translator.setSource(result.source);
187 translator.translateGraph(result.tree);
188 /*long endTime = System.nanoTime();
189 reportTime("Trans " + result.source.getName(), beginTime, endTime);
193 /*for(ISource source : orderedSources) {
194 System.out.println(source.getName());
195 GraphTranslator translator = translators.get(source);
196 translator.getVariables().forEachEntry(new TObjectIntProcedure<String>() {
198 public boolean execute(String a, int b) {
199 System.out.println(" " + a + " " + b);
205 final VariableStore variableStore = new VariableStore();
206 store.addStore(VariableStore.class, variableStore);
208 translators.forEachEntry(new TObjectObjectProcedure<ISource, GraphTranslator>() {
210 public boolean execute(ISource a, GraphTranslator b) {
211 final ArrayList<Variable> variables = new ArrayList<Variable>();
212 b.getVariables().forEachEntry(new TObjectIntProcedure<String>() {
214 public boolean execute(String a, int b) {
215 variables.add(new Variable(a, b));
219 variableStore.addSourceFile(
220 new SourceFile(a.getName(),
222 b.getDefinitionPositions()));
226 } catch(Exception e) {
228 ByteArrayOutputStream stream = new ByteArrayOutputStream();
229 e.printStackTrace(new PrintStream(stream));
230 String description = "Internal error: " +
231 new String(stream.toByteArray());
232 for(ISource source : sources)
233 errors.add(new Problem(new Location(source), description));