1 package org.simantics.scl.compiler.module.repository;
3 import java.util.ArrayList;
4 import java.util.Collection;
5 import java.util.HashMap;
7 import java.util.concurrent.ConcurrentHashMap;
9 import org.simantics.scl.compiler.common.exceptions.InternalCompilerError;
10 import org.simantics.scl.compiler.compilation.CompilationContext;
11 import org.simantics.scl.compiler.elaboration.modules.SCLValue;
12 import org.simantics.scl.compiler.environment.ConcreteEnvironment;
13 import org.simantics.scl.compiler.environment.EmptyEnvironment;
14 import org.simantics.scl.compiler.environment.Environment;
15 import org.simantics.scl.compiler.environment.NamespaceImpl.ModuleImport;
16 import org.simantics.scl.compiler.environment.NamespaceSpec;
17 import org.simantics.scl.compiler.environment.filter.NamespaceFilter;
18 import org.simantics.scl.compiler.environment.filter.NamespaceFilters;
19 import org.simantics.scl.compiler.environment.specification.EnvironmentSpecification;
20 import org.simantics.scl.compiler.errors.CompilationError;
21 import org.simantics.scl.compiler.errors.DoesNotExist;
22 import org.simantics.scl.compiler.errors.Failable;
23 import org.simantics.scl.compiler.errors.Failure;
24 import org.simantics.scl.compiler.errors.Locations;
25 import org.simantics.scl.compiler.errors.Success;
26 import org.simantics.scl.compiler.module.ImportDeclaration;
27 import org.simantics.scl.compiler.module.Module;
28 import org.simantics.scl.compiler.module.options.ModuleCompilationOptionsAdvisor;
29 import org.simantics.scl.compiler.module.repository.UpdateListener.Observable;
30 import org.simantics.scl.compiler.runtime.RuntimeEnvironment;
31 import org.simantics.scl.compiler.runtime.RuntimeEnvironmentImpl;
32 import org.simantics.scl.compiler.runtime.RuntimeModule;
33 import org.simantics.scl.compiler.runtime.RuntimeModuleMap;
34 import org.simantics.scl.compiler.source.ModuleSource;
35 import org.simantics.scl.compiler.source.repository.ModuleSourceRepository;
36 import org.simantics.scl.compiler.top.ModuleInitializer;
37 import org.simantics.scl.compiler.top.SCLCompilerConfiguration;
38 import org.simantics.scl.compiler.top.ValueNotFound;
39 import org.simantics.scl.compiler.types.Types;
40 import org.slf4j.Logger;
41 import org.slf4j.LoggerFactory;
43 import gnu.trove.map.hash.THashMap;
44 import gnu.trove.map.hash.TObjectLongHashMap;
45 import gnu.trove.procedure.TObjectObjectProcedure;
46 import gnu.trove.set.hash.THashSet;
49 * Manages compilation and caching of SCL modules.
51 * @author Hannu Niemistö
53 public class ModuleRepository {
55 private static final Logger LOGGER = LoggerFactory.getLogger(ModuleRepository.class);
57 private final ModuleRepository parentRepository;
58 private final ModuleSourceRepository sourceRepository;
59 private ConcurrentHashMap<String, ModuleEntry> moduleCache = new ConcurrentHashMap<String, ModuleEntry>();
61 private static final ThreadLocal<THashSet<String>> PENDING_MODULES = new ThreadLocal<THashSet<String>>();
63 private ModuleCompilationOptionsAdvisor advisor = null;
65 private static void beginModuleCompilation(String moduleName) {
66 THashSet<String> set = PENDING_MODULES.get();
68 set = new THashSet<String>();
69 PENDING_MODULES.set(set);
71 if(!set.add(moduleName))
72 throw new IllegalArgumentException("Cyclic module dependency detected at " + moduleName + ".");
75 private static void finishModuleCompilation(String moduleName) {
76 PENDING_MODULES.get().remove(moduleName);
79 private class ModuleEntry extends UpdateListener implements Observable {
80 final String moduleName;
81 THashSet<UpdateListener> listeners = new THashSet<UpdateListener>(); // listeners == null is used as a marker that this entry is disposed
82 // should be handled only inside synchronized code
85 Failable<Module> compilationResult;
86 Failable<RuntimeModule> runtimeModule; // created lazily
88 public ModuleEntry(String moduleName) {
89 this.moduleName = moduleName;
92 synchronized void addListener(UpdateListener listener) {
93 if(listener == null || listeners == null)
95 listeners.add(listener);
96 listener.addObservable(this);
99 public synchronized void removeListener(UpdateListener listener) {
100 if (listeners == null)
102 listeners.remove(listener);
106 public void notifyAboutUpdate() {
107 // There is a chance that another observable calls notifyAboutUpdate() before stopListening has been completed,
108 // but notifyAboutUpdate(ArrayList<UpdateListener>) lets only one thread to do the notification of dependencies
109 // by clearing listeners field.
111 ArrayList<UpdateListener> externalListeners = new ArrayList<UpdateListener>();
112 notifyAboutUpdate(externalListeners);
113 for(UpdateListener listener : externalListeners)
114 listener.notifyAboutUpdate();
117 void notifyAboutUpdate(ArrayList<UpdateListener> externalListeners) {
118 THashSet<UpdateListener> listenersCopy;
120 listenersCopy = listeners;
121 if (listenersCopy == null)
125 if(moduleCache.get(moduleName) == this) {
126 moduleCache.remove(moduleName);
127 if(SCLCompilerConfiguration.TRACE_MODULE_UPDATE) {
128 LOGGER.info("Invalidate " + moduleName);
129 for(UpdateListener l : listenersCopy)
130 LOGGER.info(" " + l);
132 for(UpdateListener l : listenersCopy)
133 if(!l.stopListening())
135 else if(l instanceof ModuleEntry)
136 ((ModuleEntry)l).notifyAboutUpdate(externalListeners);
138 externalListeners.add(l);
142 private ModuleEntry initModuleEntryAndAddListener(UpdateListener listener) {
143 source = sourceRepository.getModuleSource(moduleName, this);
146 compilationResult = DoesNotExist.getInstance();
148 if(SCLCompilerConfiguration.TRACE_MODULE_UPDATE)
149 LOGGER.info("Compile " + source);
150 beginModuleCompilation(moduleName);
151 compilationResult = source.compileModule(ModuleRepository.this, this, advisor == null ? null : advisor.getOptions(moduleName));
152 finishModuleCompilation(moduleName);
155 ModuleEntry oldEntry = moduleCache.putIfAbsent(moduleName, this);
156 if(oldEntry != null) {
157 oldEntry.addListener(listener);
161 addListener(listener);
165 @SuppressWarnings({ "rawtypes", "unchecked" })
166 public synchronized Failable<RuntimeModule> getRuntimeModule() {
167 if(runtimeModule == null) {
168 if(compilationResult.didSucceed()) {
169 Module module = compilationResult.getResult();
170 RuntimeModuleMap parentModules = new RuntimeModuleMap();
171 if(!moduleName.equals(Types.BUILTIN)) {
172 parentModules.add(ModuleRepository.this.getRuntimeModule(Types.BUILTIN)
174 Collection<ImportDeclaration> dependencies = module.getDependencies();
175 THashMap<String, ModuleEntry> moduleEntries;
177 moduleEntries = getModuleEntries(null, dependencies.toArray(new ImportDeclaration[dependencies.size()]), null, false);
178 } catch (ImportFailureException e) {
179 throw new InternalCompilerError(e);
181 for(RuntimeModule m : mapEntriesToRuntimeModules(moduleEntries).values())
182 parentModules.add(m);
184 /*for(ImportDeclaration importAst : module.getDependencies()) {
185 RuntimeModule parentModule =
186 ModuleRepository.this.getRuntimeModule(importAst.moduleName)
188 if(parentModule != null)
189 parentModules.add(parentModule);
191 RuntimeModule rm = new RuntimeModule(module, parentModules, module.getParentClassLoader());
192 ModuleInitializer initializer = module.getModuleInitializer();
193 if(initializer != null)
195 initializer.initializeModule(rm.getMutableClassLoader().getClassLoader());
196 } catch (Exception e) {
197 compilationResult = new Failure(new CompilationError[] {new CompilationError("Initialization of module " + moduleName + " failed: " + e.getMessage())});
200 runtimeModule = new Success<RuntimeModule>(rm);
203 runtimeModule = (Failable<RuntimeModule>)(Failable)compilationResult;
205 return runtimeModule;
208 public synchronized void dispose() {
212 compilationResult = null;
213 if (runtimeModule != null) {
214 if (runtimeModule.didSucceed())
215 runtimeModule.getResult().dispose();
217 runtimeModule = null;
221 public String toString() {
222 return "ModuleEntry@" + moduleName + "@" + hashCode();
226 public ModuleRepository(ModuleRepository parentRepository, ModuleSourceRepository sourceRepository) {
227 this.parentRepository = parentRepository;
228 this.sourceRepository = sourceRepository;
231 public ModuleRepository(ModuleSourceRepository sourceRepository) {
232 this(null, sourceRepository);
235 public Failable<Module> getModule(String moduleName, UpdateListener listener) {
236 return getModuleEntry(moduleName, listener).compilationResult;
239 public Failable<Module> getModule(String moduleName) {
240 return getModule(moduleName, null);
243 public void update(String moduleName) {
244 getModuleEntry(moduleName, null).notifyAboutUpdate();
247 public Failable<RuntimeModule> getRuntimeModule(String moduleName, UpdateListener listener) {
248 return getModuleEntry(moduleName, listener).getRuntimeModule();
251 public Failable<RuntimeModule> getRuntimeModule(String moduleName) {
252 return getRuntimeModule(moduleName, null);
255 private ModuleEntry getModuleEntry(String moduleName, UpdateListener listener) {
256 /* It is deliberate that the following code does not try to prevent
257 * simultaneous compilation of the same module. This is because in
258 * some situations only certain thread trying compilation can succeed
261 ModuleEntry entry = moduleCache.get(moduleName);
263 entry = new ModuleEntry(moduleName).initModuleEntryAndAddListener(listener);
265 entry.addListener(listener);
267 if(entry.compilationResult == DoesNotExist.INSTANCE && parentRepository != null)
268 return parentRepository.getModuleEntry(moduleName, listener);
273 private THashMap<String, ModuleEntry> getModuleEntries(
274 CompilationContext compilationContext,
275 ImportDeclaration[] imports,
276 UpdateListener listener,
277 boolean robustly) throws ImportFailureException {
278 THashMap<String, ModuleEntry> result = new THashMap<String, ModuleEntry>();
279 Collection<ImportFailure> failures = null;
281 TObjectLongHashMap<String> originalImports = new TObjectLongHashMap<String>();
282 ArrayList<ImportDeclaration> stack = new ArrayList<ImportDeclaration>(imports.length);
283 for(ImportDeclaration import_ : imports) {
285 originalImports.put(import_.moduleName, import_.location);
287 while(!stack.isEmpty()) {
288 ImportDeclaration import_ = stack.remove(stack.size()-1);
289 if(!result.containsKey(import_.moduleName)) {
290 boolean originalImport = originalImports.contains(import_.moduleName);
291 ModuleEntry entry = getModuleEntry(import_.moduleName, originalImport ? listener : null);
292 Failable<Module> compilationResult = entry.compilationResult;
293 if(compilationResult.didSucceed()) {
294 result.put(import_.moduleName, entry);
295 stack.addAll(compilationResult.getResult().getDependencies());
297 String deprecation = compilationResult.getResult().getDeprecation();
298 if(deprecation != null && compilationContext != null) {
299 long location = originalImport ? originalImports.get(import_.moduleName) : Locations.NO_LOCATION;
300 compilationContext.errorLog.logWarning(location, "Deprecated module " + import_.moduleName + (deprecation.isEmpty() ? "." : ": " + deprecation));
306 failures = new ArrayList<ImportFailure>(2);
307 failures.add(new ImportFailure(import_.location, import_.moduleName,
308 compilationResult == DoesNotExist.INSTANCE
309 ? ImportFailure.MODULE_DOES_NOT_EXIST_REASON
310 : ((Failure)compilationResult).errors));
315 if(failures != null && !robustly)
316 throw new ImportFailureException(failures);
321 private static THashMap<String, Module> mapEntriesToModules(THashMap<String, ModuleEntry> entries) {
322 final THashMap<String, Module> result = new THashMap<String, Module>(entries.size());
323 entries.forEachEntry(new TObjectObjectProcedure<String, ModuleEntry>() {
325 public boolean execute(String a, ModuleEntry b) {
326 result.put(a, b.compilationResult.getResult());
333 private static THashMap<String, RuntimeModule> mapEntriesToRuntimeModules(THashMap<String, ModuleEntry> entries) {
334 final THashMap<String, RuntimeModule> result = new THashMap<String, RuntimeModule>(entries.size());
335 entries.forEachEntry(new TObjectObjectProcedure<String, ModuleEntry>() {
337 public boolean execute(String a, ModuleEntry b) {
338 result.put(a, b.getRuntimeModule().getResult());
345 public Environment createEnvironment(
346 ImportDeclaration[] imports,
347 UpdateListener listener) throws ImportFailureException {
348 return createEnvironment(null, imports, listener);
351 public Environment createEnvironment(
352 CompilationContext compilationContext,
353 ImportDeclaration[] imports,
354 UpdateListener listener) throws ImportFailureException {
355 THashMap<String, ModuleEntry> entries = getModuleEntries(compilationContext, imports, listener, false);
356 THashMap<String, Module> moduleMap = mapEntriesToModules(entries);
357 return createEnvironment(moduleMap, imports);
360 public Environment createEnvironmentRobustly(
361 CompilationContext compilationContext,
362 ImportDeclaration[] imports,
363 UpdateListener listener) {
365 THashMap<String, ModuleEntry> entries = getModuleEntries(compilationContext, imports, listener, true);
366 THashMap<String, Module> moduleMap = mapEntriesToModules(entries);
367 return createEnvironment(moduleMap, imports);
368 } catch(ImportFailureException e) {
369 // Should not happen because of robust flag
370 return EmptyEnvironment.INSTANCE;
374 public Environment createEnvironment(
375 EnvironmentSpecification specification,
376 UpdateListener listener) throws ImportFailureException {
377 return createEnvironment(specification.imports.toArray(new ImportDeclaration[specification.imports.size()]), listener);
380 public RuntimeEnvironment createRuntimeEnvironment(
381 EnvironmentSpecification environmentSpecification, ClassLoader parentClassLoader) throws ImportFailureException {
382 return createRuntimeEnvironment(environmentSpecification, parentClassLoader, null);
385 public RuntimeEnvironment createRuntimeEnvironment(EnvironmentSpecification environmentSpecification) throws ImportFailureException {
386 return createRuntimeEnvironment(environmentSpecification, getClass().getClassLoader());
389 public RuntimeEnvironment createRuntimeEnvironment(
390 EnvironmentSpecification environmentSpecification,
391 ClassLoader parentClassLoader,
392 UpdateListener listener) throws ImportFailureException {
393 return createRuntimeEnvironment(
394 environmentSpecification.imports.toArray(new ImportDeclaration[environmentSpecification.imports.size()]),
399 public RuntimeEnvironment createRuntimeEnvironment(
400 ImportDeclaration[] imports,
401 ClassLoader parentClassLoader,
402 UpdateListener listener) throws ImportFailureException {
403 THashMap<String, ModuleEntry> entries = getModuleEntries(null, imports, listener, false);
404 THashMap<String, Module> moduleMap = mapEntriesToModules(entries);
405 Environment environment = createEnvironment(moduleMap, imports);
406 THashMap<String, RuntimeModule> runtimeModuleMap = mapEntriesToRuntimeModules(entries);
407 return new RuntimeEnvironmentImpl(environment, parentClassLoader, runtimeModuleMap);
410 private static Environment createEnvironment(
411 THashMap<String, Module> moduleMap,
412 ImportDeclaration[] imports) {
413 NamespaceSpec spec = new NamespaceSpec();
414 for(ImportDeclaration import_ : imports)
415 if(import_.localName != null)
416 addToNamespace(moduleMap, spec, import_.moduleName, import_.localName,
417 NamespaceFilters.createFromSpec(import_.spec));
419 return new ConcreteEnvironment(moduleMap, spec.toNamespace());
422 private static void addToNamespace(THashMap<String, Module> moduleMap,
423 NamespaceSpec namespace, String moduleName, String localName,
424 NamespaceFilter filter) {
425 if(localName.isEmpty())
426 addToNamespace(moduleMap, namespace, moduleName, filter);
428 addToNamespace(moduleMap, namespace.getNamespace(localName), moduleName, filter);
431 private static void addToNamespace(THashMap<String, Module> moduleMap,
432 NamespaceSpec namespace, String moduleName, NamespaceFilter filter) {
433 ModuleImport moduleImport = namespace.moduleMap.get(moduleName);
434 if(moduleImport == null) {
435 Module module = moduleMap.get(moduleName);
436 namespace.moduleMap.put(moduleName, new ModuleImport(module, filter));
437 for(ImportDeclaration import_ : module.getDependencies())
438 if(import_.localName != null) {
439 NamespaceFilter localFilter = NamespaceFilters.createFromSpec(import_.spec);
440 if(import_.localName.equals(""))
441 localFilter = NamespaceFilters.intersection(filter, localFilter);
442 addToNamespace(moduleMap, namespace, import_.moduleName, import_.localName, localFilter);
445 else if(!filter.isSubsetOf(moduleImport.filter)) {
446 moduleImport.filter = NamespaceFilters.union(moduleImport.filter, filter);
447 for(ImportDeclaration import_ : moduleImport.module.getDependencies())
448 // We have to recheck only modules imported to this namespace
449 if("".equals(import_.localName)) {
450 NamespaceFilter localFilter = NamespaceFilters.createFromSpec(import_.spec);
451 localFilter = NamespaceFilters.intersection(filter, localFilter);
452 addToNamespace(moduleMap, namespace, import_.moduleName, import_.localName, localFilter);
457 public Object getValue(String moduleName, String valueName) throws ValueNotFound {
458 Failable<RuntimeModule> module = getRuntimeModule(moduleName);
459 if(module.didSucceed())
460 return module.getResult().getValue(valueName);
461 else if(module == DoesNotExist.INSTANCE)
462 throw new ValueNotFound("Didn't find module " + moduleName);
464 throw new ValueNotFound(((Failure)module).toString());
467 public Object getValue(String fullValueName) throws ValueNotFound {
468 int p = fullValueName.lastIndexOf('/');
470 throw new ValueNotFound(fullValueName + " is not a valid full value name.");
471 return getValue(fullValueName.substring(0, p), fullValueName.substring(p+1));
474 public SCLValue getValueRef(String moduleName, String valueName) throws ValueNotFound {
475 Failable<Module> module = getModule(moduleName);
476 if(module.didSucceed()) {
477 SCLValue value = module.getResult().getValue(valueName);
479 throw new ValueNotFound("Module " + moduleName + " does not contain value " + valueName + ".");
482 else if(module == DoesNotExist.INSTANCE)
483 throw new ValueNotFound("Didn't find module " + moduleName);
485 throw new ValueNotFound(((Failure)module).toString());
488 public SCLValue getValueRef(String fullValueName) throws ValueNotFound {
489 int p = fullValueName.lastIndexOf('/');
491 throw new ValueNotFound(fullValueName + " is not a valid full value name.");
492 return getValueRef(fullValueName.substring(0, p), fullValueName.substring(p+1));
495 public ModuleSourceRepository getSourceRepository() {
496 return sourceRepository;
499 public String getDocumentation(String documentationName) {
500 String documentation = sourceRepository.getDocumentation(documentationName);
501 if(documentation == null && parentRepository != null)
502 return parentRepository.getDocumentation(documentationName);
503 return documentation;
507 * Flush clears module repository cache completely. It should not be called in
508 * normal operation, but may be useful during testing for clearing repositories
509 * that are statically defined.
511 public void flush() {
512 if (parentRepository != null)
513 parentRepository.flush();
514 if (moduleCache != null)
515 for (ModuleEntry entry : moduleCache.values())
517 moduleCache = new ConcurrentHashMap<String, ModuleEntry>();
521 * Gets the map of all modules that have been currently compiled successfully.
522 * Not that the method does not return all possible modules in the source repository.
524 public Map<String, Module> getModules() {
525 Map<String, Module> result = new HashMap<>(moduleCache.size());
526 for (Map.Entry<String, ModuleEntry> entry : moduleCache.entrySet()) {
527 ModuleEntry moduleEntry = entry.getValue();
528 if (moduleEntry.compilationResult.didSucceed()) {
529 result.put(entry.getKey(), moduleEntry.compilationResult.getResult());
535 public ModuleCompilationOptionsAdvisor getAdvisor() {
539 public void setAdvisor(ModuleCompilationOptionsAdvisor advisor) {
540 this.advisor = advisor;