bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/module/repository/ModuleRepository.java

   1 package org.simantics.scl.compiler.module.repository;
   2
   3 import java.util.ArrayList;
   4 import java.util.Collection;
   5 import java.util.HashMap;
   6 import java.util.Map;
   7 import java.util.concurrent.ConcurrentHashMap;
   8
   9 import org.simantics.scl.compiler.common.exceptions.InternalCompilerError;
  10 import org.simantics.scl.compiler.compilation.CompilationContext;
  11 import org.simantics.scl.compiler.elaboration.modules.SCLValue;
  12 import org.simantics.scl.compiler.environment.ConcreteEnvironment;
  13 import org.simantics.scl.compiler.environment.EmptyEnvironment;
  14 import org.simantics.scl.compiler.environment.Environment;
  15 import org.simantics.scl.compiler.environment.NamespaceImpl.ModuleImport;
  16 import org.simantics.scl.compiler.environment.NamespaceSpec;
  17 import org.simantics.scl.compiler.environment.filter.NamespaceFilter;
  18 import org.simantics.scl.compiler.environment.filter.NamespaceFilters;
  19 import org.simantics.scl.compiler.environment.specification.EnvironmentSpecification;
  20 import org.simantics.scl.compiler.errors.CompilationError;
  21 import org.simantics.scl.compiler.errors.DoesNotExist;
  22 import org.simantics.scl.compiler.errors.Failable;
  23 import org.simantics.scl.compiler.errors.Failure;
  24 import org.simantics.scl.compiler.errors.Locations;
  25 import org.simantics.scl.compiler.errors.Success;
  26 import org.simantics.scl.compiler.module.ImportDeclaration;
  27 import org.simantics.scl.compiler.module.Module;
  28 import org.simantics.scl.compiler.module.options.ModuleCompilationOptionsAdvisor;
  29 import org.simantics.scl.compiler.module.repository.UpdateListener.Observable;
  30 import org.simantics.scl.compiler.runtime.RuntimeEnvironment;
  31 import org.simantics.scl.compiler.runtime.RuntimeEnvironmentImpl;
  32 import org.simantics.scl.compiler.runtime.RuntimeModule;
  33 import org.simantics.scl.compiler.runtime.RuntimeModuleMap;
  34 import org.simantics.scl.compiler.source.ModuleSource;
  35 import org.simantics.scl.compiler.source.repository.ModuleSourceRepository;
  36 import org.simantics.scl.compiler.top.ModuleInitializer;
  37 import org.simantics.scl.compiler.top.SCLCompilerConfiguration;
  38 import org.simantics.scl.compiler.top.ValueNotFound;
  39 import org.simantics.scl.compiler.types.Types;
  40
  41 import gnu.trove.map.hash.THashMap;
  42 import gnu.trove.map.hash.TObjectLongHashMap;
  43 import gnu.trove.procedure.TObjectObjectProcedure;
  44 import gnu.trove.set.hash.THashSet;
  45
  46 /**
  47  * Manages compilation and caching of SCL modules.
  48  *
  49  * @author Hannu Niemist&ouml;
  50  */
  51 public class ModuleRepository {
  52     private final ModuleRepository parentRepository;
  53     private final ModuleSourceRepository sourceRepository;
  54     private ConcurrentHashMap<String, ModuleEntry> moduleCache = new ConcurrentHashMap<String, ModuleEntry>();
  55
  56     private static final ThreadLocal<THashSet<String>> PENDING_MODULES = new ThreadLocal<THashSet<String>>();
  57
  58     private ModuleCompilationOptionsAdvisor advisor = null;
  59
  60     private static void beginModuleCompilation(String moduleName) {
  61         THashSet<String> set = PENDING_MODULES.get();
  62         if(set == null) {
  63             set = new THashSet<String>();
  64             PENDING_MODULES.set(set);
  65         }
  66         if(!set.add(moduleName))
  67             throw new IllegalArgumentException("Cyclic module dependency detected at " + moduleName + ".");
  68     }
  69
  70     private static void finishModuleCompilation(String moduleName) {
  71         PENDING_MODULES.get().remove(moduleName);
  72     }
  73
  74     private class ModuleEntry extends UpdateListener implements Observable {
  75         final String moduleName;
  76         THashSet<UpdateListener> listeners = new THashSet<UpdateListener>();
  77
  78         ModuleSource source;
  79         Failable<Module> compilationResult;
  80         Failable<RuntimeModule> runtimeModule; // created lazily
  81
  82         public ModuleEntry(String moduleName) {
  83             this.moduleName = moduleName;
  84         }
  85
  86         synchronized void addListener(UpdateListener listener) {
  87             if(listener == null || listeners == null)
  88                 return;
  89             listeners.add(listener);
  90             listener.addObservable(this);
  91         }
  92
  93         public synchronized void removeListener(UpdateListener listener) {
  94             if (listeners == null)
  95                 return;
  96             listeners.remove(listener);
  97         }
  98
  99         @Override
 100         public void notifyAboutUpdate() {
 101             ArrayList<UpdateListener> externalListeners = new ArrayList<UpdateListener>();
 102             notifyAboutUpdate(externalListeners);
 103             for(UpdateListener listener : externalListeners)
 104                 listener.notifyAboutUpdate();
 105         }
 106
 107         synchronized void notifyAboutUpdate(ArrayList<UpdateListener> externalListeners) {
 108             stopListening();
 109             if (listeners == null)
 110                 return;
 111             if(moduleCache.get(moduleName) == this) {
 112                 moduleCache.remove(moduleName);
 113                 if(SCLCompilerConfiguration.TRACE_MODULE_UPDATE) {
 114                     System.out.println("Invalidate " + moduleName);
 115                     for(UpdateListener l : listeners)
 116                         System.out.println("    " + l);
 117                 }
 118                 THashSet<UpdateListener> listenersCopy = listeners;
 119                 listeners = null;
 120                 for(UpdateListener l : listenersCopy)
 121                     l.stopListening();
 122                 for(UpdateListener l : listenersCopy)
 123                     if(l instanceof ModuleEntry)
 124                         ((ModuleEntry)l).notifyAboutUpdate(externalListeners);
 125                     else {
 126                         externalListeners.add(l);
 127                     }
 128             }
 129         }
 130
 131         private ModuleEntry initModuleEntryAndAddListener(UpdateListener listener) {
 132             source = sourceRepository.getModuleSource(moduleName, this);
 133
 134             if(source == null)
 135                 compilationResult = DoesNotExist.getInstance();
 136             else {
 137                 if(SCLCompilerConfiguration.TRACE_MODULE_UPDATE)
 138                     System.out.println("Compile " + source);
 139                 beginModuleCompilation(moduleName);
 140                 compilationResult = source.compileModule(ModuleRepository.this, this, advisor == null ? null : advisor.getOptions(moduleName));
 141                 finishModuleCompilation(moduleName);
 142             }
 143
 144             ModuleEntry oldEntry = moduleCache.putIfAbsent(moduleName, this);
 145             if(oldEntry != null) {
 146                 oldEntry.addListener(listener);
 147                 return oldEntry;
 148             }
 149
 150             addListener(listener);
 151             return this;
 152         }
 153
 154         @SuppressWarnings({ "rawtypes", "unchecked" })
 155         public synchronized Failable<RuntimeModule> getRuntimeModule() {
 156             if(runtimeModule == null) {
 157                 if(compilationResult.didSucceed()) {
 158                     Module module = compilationResult.getResult();
 159                     RuntimeModuleMap parentModules = new RuntimeModuleMap();
 160                     if(!moduleName.equals(Types.BUILTIN)) {
 161                         parentModules.add(ModuleRepository.this.getRuntimeModule(Types.BUILTIN)
 162                                 .getResult());
 163                         Collection<ImportDeclaration> dependencies = module.getDependencies();
 164                         THashMap<String, ModuleEntry> moduleEntries;
 165                         try {
 166                             moduleEntries = getModuleEntries(null, dependencies.toArray(new ImportDeclaration[dependencies.size()]), null, false);
 167                         } catch (ImportFailureException e) {
 168                             throw new InternalCompilerError(e);
 169                         }
 170                         for(RuntimeModule m : mapEntriesToRuntimeModules(moduleEntries).values())
 171                             parentModules.add(m);
 172                     }
 173                     /*for(ImportDeclaration importAst : module.getDependencies()) {
 174                         RuntimeModule parentModule =
 175                                 ModuleRepository.this.getRuntimeModule(importAst.moduleName)
 176                                 .getResult();
 177                         if(parentModule != null)
 178                             parentModules.add(parentModule);
 179                     }*/
 180                     RuntimeModule rm = new RuntimeModule(module, parentModules, module.getParentClassLoader());
 181                     ModuleInitializer initializer = module.getModuleInitializer();
 182                     if(initializer != null)
 183                         try {
 184                             initializer.initializeModule(rm.getMutableClassLoader().getClassLoader());
 185                         } catch (Exception e) {
 186                             compilationResult = new Failure(new CompilationError[] {new CompilationError("Initialization of module " + moduleName + " failed: " + e.getMessage())});
 187                             e.printStackTrace();
 188                         }
 189                     runtimeModule = new Success<RuntimeModule>(rm);
 190                 }
 191                 else
 192                     runtimeModule = (Failable<RuntimeModule>)(Failable)compilationResult;
 193             }
 194             return runtimeModule;
 195         }
 196
 197         public synchronized void dispose() {
 198             if (listeners != null)
 199                 listeners.clear();
 200             listeners = null;
 201             stopListening();
 202             source = null;
 203             compilationResult = null;
 204             if (runtimeModule != null) {
 205                 if (runtimeModule.didSucceed())
 206                     runtimeModule.getResult().dispose();
 207             }
 208             runtimeModule = null;
 209         }
 210
 211         @Override
 212         public String toString() {
 213             return "ModuleEntry@" + moduleName + "@" + hashCode();
 214         }
 215     }
 216
 217     public ModuleRepository(ModuleRepository parentRepository, ModuleSourceRepository sourceRepository) {
 218         this.parentRepository = parentRepository;
 219         this.sourceRepository = sourceRepository;
 220     }
 221
 222     public ModuleRepository(ModuleSourceRepository sourceRepository) {
 223         this(null, sourceRepository);
 224     }
 225
 226     public Failable<Module> getModule(String moduleName, UpdateListener listener) {
 227         return getModuleEntry(moduleName, listener).compilationResult;
 228     }
 229
 230     public Failable<Module> getModule(String moduleName) {
 231         return getModule(moduleName, null);
 232     }
 233
 234     public Failable<RuntimeModule> getRuntimeModule(String moduleName, UpdateListener listener) {
 235         return getModuleEntry(moduleName, listener).getRuntimeModule();
 236     }
 237
 238     public Failable<RuntimeModule> getRuntimeModule(String moduleName) {
 239         return getRuntimeModule(moduleName, null);
 240     }
 241
 242     private ModuleEntry getModuleEntry(String moduleName, UpdateListener listener) {
 243         /* It is deliberate that the following code does not try to prevent
 244          * simultaneous compilation of the same module. This is because in
 245          * some situations only certain thread trying compilation can succeed
 246          * in it.
 247          */
 248         ModuleEntry entry = moduleCache.get(moduleName);
 249         if(entry == null)
 250             entry = new ModuleEntry(moduleName).initModuleEntryAndAddListener(listener);
 251         else
 252             entry.addListener(listener);
 253
 254         if(entry.compilationResult == DoesNotExist.INSTANCE && parentRepository != null)
 255             return parentRepository.getModuleEntry(moduleName, listener);
 256         else
 257             return entry;
 258     }
 259
 260     private THashMap<String, ModuleEntry> getModuleEntries(
 261             CompilationContext compilationContext,
 262             ImportDeclaration[] imports,
 263             UpdateListener listener,
 264             boolean robustly) throws ImportFailureException {
 265         THashMap<String, ModuleEntry> result = new THashMap<String, ModuleEntry>();
 266         Collection<ImportFailure> failures = null;
 267
 268         TObjectLongHashMap<String> originalImports = new TObjectLongHashMap<String>();
 269         ArrayList<ImportDeclaration> stack = new ArrayList<ImportDeclaration>(imports.length);
 270         for(ImportDeclaration import_ : imports) {
 271             stack.add(import_);
 272             originalImports.put(import_.moduleName, import_.location);
 273         }
 274         while(!stack.isEmpty()) {
 275             ImportDeclaration import_ = stack.remove(stack.size()-1);
 276             if(!result.containsKey(import_.moduleName)) {
 277                 boolean originalImport = originalImports.contains(import_.moduleName);
 278                 ModuleEntry entry = getModuleEntry(import_.moduleName, originalImport ? listener : null);
 279                 Failable<Module> compilationResult = entry.compilationResult;
 280                 if(compilationResult.didSucceed()) {
 281                     result.put(import_.moduleName, entry);
 282                     stack.addAll(compilationResult.getResult().getDependencies());
 283                     if(originalImport) {
 284                         String deprecation = compilationResult.getResult().getDeprecation();
 285                         if(deprecation != null && compilationContext != null) {
 286                             long location = originalImport ? originalImports.get(import_.moduleName) : Locations.NO_LOCATION;
 287                             compilationContext.errorLog.logWarning(location, "Deprecated module " + import_.moduleName   + (deprecation.isEmpty() ? "." : ": " + deprecation));
 288                         }
 289                     }
 290                 }
 291                 else {
 292                     if(failures == null)
 293                         failures = new ArrayList<ImportFailure>(2);
 294                     failures.add(new ImportFailure(import_.location, import_.moduleName,
 295                             compilationResult == DoesNotExist.INSTANCE
 296                                     ? ImportFailure.MODULE_DOES_NOT_EXIST_REASON
 297                                     : ((Failure)compilationResult).errors));
 298                 }
 299             }
 300         }
 301
 302         if(failures != null && !robustly)
 303             throw new ImportFailureException(failures);
 304
 305         return result;
 306     }
 307
 308     private static THashMap<String, Module> mapEntriesToModules(THashMap<String, ModuleEntry> entries) {
 309         final THashMap<String, Module> result = new THashMap<String, Module>(entries.size());
 310         entries.forEachEntry(new TObjectObjectProcedure<String, ModuleEntry>() {
 311             @Override
 312             public boolean execute(String a, ModuleEntry b) {
 313                 result.put(a, b.compilationResult.getResult());
 314                 return true;
 315             }
 316         });
 317         return result;
 318     }
 319
 320     private static THashMap<String, RuntimeModule> mapEntriesToRuntimeModules(THashMap<String, ModuleEntry> entries) {
 321         final THashMap<String, RuntimeModule> result = new THashMap<String, RuntimeModule>(entries.size());
 322         entries.forEachEntry(new TObjectObjectProcedure<String, ModuleEntry>() {
 323             @Override
 324             public boolean execute(String a, ModuleEntry b) {
 325                 result.put(a, b.getRuntimeModule().getResult());
 326                 return true;
 327             }
 328         });
 329         return result;
 330     }
 331
 332     public Environment createEnvironment(
 333             ImportDeclaration[] imports,
 334             UpdateListener listener) throws ImportFailureException {
 335         return createEnvironment(null, imports, listener);
 336     }
 337
 338     public Environment createEnvironment(
 339             CompilationContext compilationContext,
 340             ImportDeclaration[] imports,
 341             UpdateListener listener) throws ImportFailureException {
 342         THashMap<String, ModuleEntry> entries = getModuleEntries(compilationContext, imports, listener, false);
 343         THashMap<String, Module> moduleMap = mapEntriesToModules(entries);
 344         return createEnvironment(moduleMap, imports);
 345     }
 346
 347     public Environment createEnvironmentRobustly(
 348             CompilationContext compilationContext,
 349             ImportDeclaration[] imports,
 350             UpdateListener listener) {
 351         try {
 352             THashMap<String, ModuleEntry> entries = getModuleEntries(compilationContext, imports, listener, true);
 353             THashMap<String, Module> moduleMap = mapEntriesToModules(entries);
 354             return createEnvironment(moduleMap, imports);
 355         } catch(ImportFailureException e) {
 356             // Should not happen because of robust flag
 357             return EmptyEnvironment.INSTANCE;
 358         }
 359     }
 360
 361     public Environment createEnvironment(
 362             EnvironmentSpecification specification,
 363             UpdateListener listener) throws ImportFailureException {
 364         return createEnvironment(specification.imports.toArray(new ImportDeclaration[specification.imports.size()]), listener);
 365     }
 366
 367     public RuntimeEnvironment createRuntimeEnvironment(
 368             EnvironmentSpecification environmentSpecification, ClassLoader parentClassLoader) throws ImportFailureException {
 369         return createRuntimeEnvironment(environmentSpecification, parentClassLoader, null);
 370     }
 371
 372     public RuntimeEnvironment createRuntimeEnvironment(EnvironmentSpecification environmentSpecification) throws ImportFailureException {
 373         return createRuntimeEnvironment(environmentSpecification, getClass().getClassLoader());
 374     }
 375
 376     public RuntimeEnvironment createRuntimeEnvironment(
 377             EnvironmentSpecification environmentSpecification,
 378             ClassLoader parentClassLoader,
 379             UpdateListener listener) throws ImportFailureException {
 380         return createRuntimeEnvironment(
 381                 environmentSpecification.imports.toArray(new ImportDeclaration[environmentSpecification.imports.size()]),
 382                 parentClassLoader,
 383                 listener);
 384     }
 385
 386     public RuntimeEnvironment createRuntimeEnvironment(
 387             ImportDeclaration[] imports,
 388             ClassLoader parentClassLoader,
 389             UpdateListener listener) throws ImportFailureException {
 390         THashMap<String, ModuleEntry> entries = getModuleEntries(null, imports, listener, false);
 391         THashMap<String, Module> moduleMap = mapEntriesToModules(entries);
 392         Environment environment = createEnvironment(moduleMap, imports);
 393         THashMap<String, RuntimeModule> runtimeModuleMap = mapEntriesToRuntimeModules(entries);
 394         return new RuntimeEnvironmentImpl(environment, parentClassLoader, runtimeModuleMap);
 395     }
 396
 397     private static Environment createEnvironment(
 398             THashMap<String, Module> moduleMap,
 399             ImportDeclaration[] imports) {
 400         NamespaceSpec spec = new NamespaceSpec();
 401         for(ImportDeclaration import_ : imports)
 402             if(import_.localName != null)
 403                 addToNamespace(moduleMap, spec, import_.moduleName, import_.localName,
 404                         NamespaceFilters.createFromSpec(import_.spec));
 405
 406         return new ConcreteEnvironment(moduleMap, spec.toNamespace());
 407     }
 408
 409     private static void addToNamespace(THashMap<String, Module> moduleMap,
 410             NamespaceSpec namespace, String moduleName, String localName,
 411             NamespaceFilter filter) {
 412         if(localName.isEmpty())
 413             addToNamespace(moduleMap, namespace, moduleName, filter);
 414         else
 415             addToNamespace(moduleMap, namespace.getNamespace(localName), moduleName, filter);
 416     }
 417
 418     private static void addToNamespace(THashMap<String, Module> moduleMap,
 419             NamespaceSpec namespace, String moduleName, NamespaceFilter filter) {
 420         ModuleImport moduleImport = namespace.moduleMap.get(moduleName);
 421         if(moduleImport == null) {
 422             Module module = moduleMap.get(moduleName);
 423             namespace.moduleMap.put(moduleName, new ModuleImport(module, filter));
 424             for(ImportDeclaration import_ : module.getDependencies())
 425                 if(import_.localName != null) {
 426                     NamespaceFilter localFilter = NamespaceFilters.createFromSpec(import_.spec);
 427                     if(import_.localName.equals(""))
 428                         localFilter = NamespaceFilters.intersection(filter, localFilter);
 429                     addToNamespace(moduleMap, namespace, import_.moduleName, import_.localName, localFilter);
 430                 }
 431         }
 432         else if(!filter.isSubsetOf(moduleImport.filter)) {
 433             moduleImport.filter = NamespaceFilters.union(moduleImport.filter, filter);
 434             for(ImportDeclaration import_ : moduleImport.module.getDependencies())
 435                 // We have to recheck only modules imported to this namespace
 436                 if("".equals(import_.localName)) {
 437                     NamespaceFilter localFilter = NamespaceFilters.createFromSpec(import_.spec);
 438                     localFilter = NamespaceFilters.intersection(filter, localFilter);
 439                     addToNamespace(moduleMap, namespace, import_.moduleName, import_.localName, localFilter);
 440                 }
 441         }
 442     }
 443
 444     public Object getValue(String moduleName, String valueName) throws ValueNotFound {
 445         Failable<RuntimeModule> module = getRuntimeModule(moduleName);
 446         if(module.didSucceed())
 447             return module.getResult().getValue(valueName);
 448         else if(module == DoesNotExist.INSTANCE)
 449             throw new ValueNotFound("Didn't find module " + moduleName);
 450         else
 451             throw new ValueNotFound(((Failure)module).toString());
 452     }
 453
 454     public Object getValue(String fullValueName) throws ValueNotFound {
 455         int p = fullValueName.lastIndexOf('/');
 456         if(p < 0)
 457             throw new ValueNotFound(fullValueName + " is not a valid full value name.");
 458         return getValue(fullValueName.substring(0, p), fullValueName.substring(p+1));
 459     }
 460
 461     public SCLValue getValueRef(String moduleName, String valueName) throws ValueNotFound {
 462         Failable<Module> module = getModule(moduleName);
 463         if(module.didSucceed()) {
 464             SCLValue value = module.getResult().getValue(valueName);
 465             if(value == null)
 466                 throw new ValueNotFound("Module " + moduleName + " does not contain value " + valueName + ".");
 467             return value;
 468         }
 469         else if(module == DoesNotExist.INSTANCE)
 470             throw new ValueNotFound("Didn't find module " + moduleName);
 471         else
 472             throw new ValueNotFound(((Failure)module).toString());
 473     }
 474
 475     public SCLValue getValueRef(String fullValueName) throws ValueNotFound {
 476         int p = fullValueName.lastIndexOf('/');
 477         if(p < 0)
 478             throw new ValueNotFound(fullValueName + " is not a valid full value name.");
 479         return getValueRef(fullValueName.substring(0, p), fullValueName.substring(p+1));
 480     }
 481
 482     public ModuleSourceRepository getSourceRepository() {
 483         return sourceRepository;
 484     }
 485
 486     public String getDocumentation(String documentationName) {
 487         String documentation = sourceRepository.getDocumentation(documentationName);
 488         if(documentation == null && parentRepository != null)
 489             return parentRepository.getDocumentation(documentationName);
 490         return documentation;
 491     }
 492
 493     public void flush() {
 494         if (parentRepository != null)
 495             parentRepository.flush();
 496         if (moduleCache != null) {
 497             for (ModuleEntry entry : moduleCache.values()) {
 498                 entry.dispose();
 499             }
 500             moduleCache.clear();
 501         }
 502         moduleCache = null;
 503     }
 504
 505     public Map<String, Module> getModules() {
 506         Map<String, Module> result = new HashMap<>(moduleCache.size());
 507         for (Map.Entry<String, ModuleEntry> entry : moduleCache.entrySet()) {
 508             ModuleEntry moduleEntry = entry.getValue();
 509             if (moduleEntry.compilationResult.didSucceed()) {
 510                 result.put(entry.getKey(), moduleEntry.compilationResult.getResult());
 511             }
 512         }
 513         return result;
 514     }
 515
 516     public ModuleCompilationOptionsAdvisor getAdvisor() {
 517         return advisor;
 518     }
 519
 520     public void setAdvisor(ModuleCompilationOptionsAdvisor advisor) {
 521         this.advisor = advisor;
 522     }
 523
 524 }
 525