bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/module/repository/ModuleRepository.java

   1 package org.simantics.scl.compiler.module.repository;
   2
   3 import java.util.ArrayList;
   4 import java.util.Collection;
   5 import java.util.HashMap;
   6 import java.util.HashSet;
   7 import java.util.Map;
   8 import java.util.Set;
   9 import java.util.concurrent.ConcurrentHashMap;
  10
  11 import org.simantics.scl.compiler.common.exceptions.InternalCompilerError;
  12 import org.simantics.scl.compiler.compilation.CompilationContext;
  13 import org.simantics.scl.compiler.elaboration.modules.SCLValue;
  14 import org.simantics.scl.compiler.environment.ConcreteEnvironment;
  15 import org.simantics.scl.compiler.environment.EmptyEnvironment;
  16 import org.simantics.scl.compiler.environment.Environment;
  17 import org.simantics.scl.compiler.environment.NamespaceImpl.ModuleImport;
  18 import org.simantics.scl.compiler.environment.NamespaceSpec;
  19 import org.simantics.scl.compiler.environment.filter.NamespaceFilter;
  20 import org.simantics.scl.compiler.environment.filter.NamespaceFilters;
  21 import org.simantics.scl.compiler.environment.specification.EnvironmentSpecification;
  22 import org.simantics.scl.compiler.errors.CompilationError;
  23 import org.simantics.scl.compiler.errors.DoesNotExist;
  24 import org.simantics.scl.compiler.errors.Failable;
  25 import org.simantics.scl.compiler.errors.Failure;
  26 import org.simantics.scl.compiler.errors.Locations;
  27 import org.simantics.scl.compiler.errors.Success;
  28 import org.simantics.scl.compiler.module.ImportDeclaration;
  29 import org.simantics.scl.compiler.module.Module;
  30 import org.simantics.scl.compiler.module.options.ModuleCompilationOptionsAdvisor;
  31 import org.simantics.scl.compiler.module.repository.UpdateListener.Observable;
  32 import org.simantics.scl.compiler.runtime.RuntimeEnvironment;
  33 import org.simantics.scl.compiler.runtime.RuntimeEnvironmentImpl;
  34 import org.simantics.scl.compiler.runtime.RuntimeModule;
  35 import org.simantics.scl.compiler.runtime.RuntimeModuleMap;
  36 import org.simantics.scl.compiler.source.ModuleSource;
  37 import org.simantics.scl.compiler.source.repository.ModuleSourceRepository;
  38 import org.simantics.scl.compiler.top.ModuleInitializer;
  39 import org.simantics.scl.compiler.top.SCLCompilerConfiguration;
  40 import org.simantics.scl.compiler.top.ValueNotFound;
  41 import org.simantics.scl.compiler.types.Types;
  42
  43 import gnu.trove.map.hash.THashMap;
  44 import gnu.trove.map.hash.TObjectLongHashMap;
  45 import gnu.trove.procedure.TObjectObjectProcedure;
  46 import gnu.trove.set.hash.THashSet;
  47
  48 /**
  49  * Manages compilation and caching of SCL modules.
  50  *
  51  * @author Hannu Niemist&ouml;
  52  */
  53 public class ModuleRepository {
  54     private final ModuleRepository parentRepository;
  55     private final ModuleSourceRepository sourceRepository;
  56     private ConcurrentHashMap<String, ModuleEntry> moduleCache = new ConcurrentHashMap<String, ModuleEntry>();
  57     public Set<RuntimeEnvironment> runtimeEnvironments = new HashSet<RuntimeEnvironment>();
  58
  59     private static final ThreadLocal<THashSet<String>> PENDING_MODULES = new ThreadLocal<THashSet<String>>();
  60
  61     private ModuleCompilationOptionsAdvisor advisor = null;
  62
  63     private static void beginModuleCompilation(String moduleName) {
  64         THashSet<String> set = PENDING_MODULES.get();
  65         if(set == null) {
  66             set = new THashSet<String>();
  67             PENDING_MODULES.set(set);
  68         }
  69         if(!set.add(moduleName))
  70             throw new IllegalArgumentException("Cyclic module dependency detected at " + moduleName + ".");
  71     }
  72
  73     private static void finishModuleCompilation(String moduleName) {
  74         PENDING_MODULES.get().remove(moduleName);
  75     }
  76
  77     private class ModuleEntry extends UpdateListener implements Observable {
  78         final String moduleName;
  79         THashSet<UpdateListener> listeners = new THashSet<UpdateListener>(); // listeners == null is used as a marker that this entry is disposed
  80                                                                              // should be handled only inside synchronized code
  81
  82         ModuleSource source;
  83         Failable<Module> compilationResult;
  84         Failable<RuntimeModule> runtimeModule; // created lazily
  85
  86         public ModuleEntry(String moduleName) {
  87             this.moduleName = moduleName;
  88         }
  89
  90         synchronized void addListener(UpdateListener listener) {
  91             if(listener == null || listeners == null)
  92                 return;
  93             listeners.add(listener);
  94             listener.addObservable(this);
  95         }
  96
  97         public synchronized void removeListener(UpdateListener listener) {
  98             if (listeners == null)
  99                 return;
 100             listeners.remove(listener);
 101         }
 102
 103         @Override
 104         public void notifyAboutUpdate() {
 105             // There is a chance that another observable calls notifyAboutUpdate() before stopListening has been completed,
 106             // but notifyAboutUpdate(ArrayList<UpdateListener>) lets only one thread to do the notification of dependencies
 107             // by clearing listeners field.
 108             stopListening();
 109             ArrayList<UpdateListener> externalListeners = new ArrayList<UpdateListener>();
 110             notifyAboutUpdate(externalListeners);
 111             for(UpdateListener listener : externalListeners)
 112                 listener.notifyAboutUpdate();
 113         }
 114
 115         void notifyAboutUpdate(ArrayList<UpdateListener> externalListeners) {
 116             THashSet<UpdateListener> listenersCopy;
 117             synchronized(this) {
 118                 listenersCopy = listeners;
 119                 if (listenersCopy == null)
 120                     return;
 121                 listeners = null;
 122             }
 123             if(moduleCache.get(moduleName) == this) {
 124                 moduleCache.remove(moduleName);
 125                 if(SCLCompilerConfiguration.TRACE_MODULE_UPDATE) {
 126                     System.out.println("Invalidate " + moduleName);
 127                     for(UpdateListener l : listenersCopy)
 128                         System.out.println("    " + l);
 129                 }
 130                 for(UpdateListener l : listenersCopy)
 131                     if(!l.stopListening())
 132                         ;
 133                     else if(l instanceof ModuleEntry)
 134                         ((ModuleEntry)l).notifyAboutUpdate(externalListeners);
 135                     else
 136                         externalListeners.add(l);
 137             }
 138         }
 139
 140         private ModuleEntry initModuleEntryAndAddListener(UpdateListener listener) {
 141             source = sourceRepository.getModuleSource(moduleName, this);
 142
 143             if(source == null)
 144                 compilationResult = DoesNotExist.getInstance();
 145             else {
 146                 if(SCLCompilerConfiguration.TRACE_MODULE_UPDATE)
 147                     System.out.println("Compile " + source);
 148                 beginModuleCompilation(moduleName);
 149                 compilationResult = source.compileModule(ModuleRepository.this, this, advisor == null ? null : advisor.getOptions(moduleName));
 150                 finishModuleCompilation(moduleName);
 151             }
 152
 153             ModuleEntry oldEntry = moduleCache.putIfAbsent(moduleName, this);
 154             if(oldEntry != null) {
 155                 oldEntry.addListener(listener);
 156                 return oldEntry;
 157             }
 158
 159             addListener(listener);
 160             return this;
 161         }
 162
 163         @SuppressWarnings({ "rawtypes", "unchecked" })
 164         public synchronized Failable<RuntimeModule> getRuntimeModule() {
 165             if(runtimeModule == null) {
 166                 if(compilationResult.didSucceed()) {
 167                     Module module = compilationResult.getResult();
 168                     RuntimeModuleMap parentModules = new RuntimeModuleMap();
 169                     if(!moduleName.equals(Types.BUILTIN)) {
 170                         parentModules.add(ModuleRepository.this.getRuntimeModule(Types.BUILTIN)
 171                                 .getResult());
 172                         Collection<ImportDeclaration> dependencies = module.getDependencies();
 173                         THashMap<String, ModuleEntry> moduleEntries;
 174                         try {
 175                             moduleEntries = getModuleEntries(null, dependencies.toArray(new ImportDeclaration[dependencies.size()]), null, false);
 176                         } catch (ImportFailureException e) {
 177                             throw new InternalCompilerError(e);
 178                         }
 179                         for(RuntimeModule m : mapEntriesToRuntimeModules(moduleEntries).values())
 180                             parentModules.add(m);
 181                     }
 182                     /*for(ImportDeclaration importAst : module.getDependencies()) {
 183                         RuntimeModule parentModule =
 184                                 ModuleRepository.this.getRuntimeModule(importAst.moduleName)
 185                                 .getResult();
 186                         if(parentModule != null)
 187                             parentModules.add(parentModule);
 188                     }*/
 189                     RuntimeModule rm = new RuntimeModule(module, parentModules, module.getParentClassLoader());
 190                     ModuleInitializer initializer = module.getModuleInitializer();
 191                     if(initializer != null)
 192                         try {
 193                             initializer.initializeModule(rm.getMutableClassLoader().getClassLoader());
 194                         } catch (Exception e) {
 195                             compilationResult = new Failure(new CompilationError[] {new CompilationError("Initialization of module " + moduleName + " failed: " + e.getMessage())});
 196                             e.printStackTrace();
 197                         }
 198                     runtimeModule = new Success<RuntimeModule>(rm);
 199                 }
 200                 else
 201                     runtimeModule = (Failable<RuntimeModule>)(Failable)compilationResult;
 202             }
 203             return runtimeModule;
 204         }
 205
 206         public synchronized void dispose() {
 207             listeners = null;
 208             stopListening();
 209             source = null;
 210             compilationResult = null;
 211             if (runtimeModule != null) {
 212                 if (runtimeModule.didSucceed())
 213                     runtimeModule.getResult().dispose();
 214             }
 215             runtimeModule = null;
 216         }
 217
 218         @Override
 219         public String toString() {
 220             return "ModuleEntry@" + moduleName + "@" + hashCode();
 221         }
 222     }
 223
 224     public ModuleRepository(ModuleRepository parentRepository, ModuleSourceRepository sourceRepository) {
 225         this.parentRepository = parentRepository;
 226         this.sourceRepository = sourceRepository;
 227     }
 228
 229     public ModuleRepository(ModuleSourceRepository sourceRepository) {
 230         this(null, sourceRepository);
 231     }
 232
 233     public Failable<Module> getModule(String moduleName, UpdateListener listener) {
 234         return getModuleEntry(moduleName, listener).compilationResult;
 235     }
 236
 237     public Failable<Module> getModule(String moduleName) {
 238         return getModule(moduleName, null);
 239     }
 240
 241     public Failable<RuntimeModule> getRuntimeModule(String moduleName, UpdateListener listener) {
 242         return getModuleEntry(moduleName, listener).getRuntimeModule();
 243     }
 244
 245     public Failable<RuntimeModule> getRuntimeModule(String moduleName) {
 246         return getRuntimeModule(moduleName, null);
 247     }
 248
 249     private ModuleEntry getModuleEntry(String moduleName, UpdateListener listener) {
 250         /* It is deliberate that the following code does not try to prevent
 251          * simultaneous compilation of the same module. This is because in
 252          * some situations only certain thread trying compilation can succeed
 253          * in it.
 254          */
 255         ModuleEntry entry = moduleCache.get(moduleName);
 256         if(entry == null)
 257             entry = new ModuleEntry(moduleName).initModuleEntryAndAddListener(listener);
 258         else
 259             entry.addListener(listener);
 260
 261         if(entry.compilationResult == DoesNotExist.INSTANCE && parentRepository != null)
 262             return parentRepository.getModuleEntry(moduleName, listener);
 263         else
 264             return entry;
 265     }
 266
 267     private THashMap<String, ModuleEntry> getModuleEntries(
 268             CompilationContext compilationContext,
 269             ImportDeclaration[] imports,
 270             UpdateListener listener,
 271             boolean robustly) throws ImportFailureException {
 272         THashMap<String, ModuleEntry> result = new THashMap<String, ModuleEntry>();
 273         Collection<ImportFailure> failures = null;
 274
 275         TObjectLongHashMap<String> originalImports = new TObjectLongHashMap<String>();
 276         ArrayList<ImportDeclaration> stack = new ArrayList<ImportDeclaration>(imports.length);
 277         for(ImportDeclaration import_ : imports) {
 278             stack.add(import_);
 279             originalImports.put(import_.moduleName, import_.location);
 280         }
 281         while(!stack.isEmpty()) {
 282             ImportDeclaration import_ = stack.remove(stack.size()-1);
 283             if(!result.containsKey(import_.moduleName)) {
 284                 boolean originalImport = originalImports.contains(import_.moduleName);
 285                 ModuleEntry entry = getModuleEntry(import_.moduleName, originalImport ? listener : null);
 286                 Failable<Module> compilationResult = entry.compilationResult;
 287                 if(compilationResult.didSucceed()) {
 288                     result.put(import_.moduleName, entry);
 289                     stack.addAll(compilationResult.getResult().getDependencies());
 290                     if(originalImport) {
 291                         String deprecation = compilationResult.getResult().getDeprecation();
 292                         if(deprecation != null && compilationContext != null) {
 293                             long location = originalImport ? originalImports.get(import_.moduleName) : Locations.NO_LOCATION;
 294                             compilationContext.errorLog.logWarning(location, "Deprecated module " + import_.moduleName   + (deprecation.isEmpty() ? "." : ": " + deprecation));
 295                         }
 296                     }
 297                 }
 298                 else {
 299                     if(failures == null)
 300                         failures = new ArrayList<ImportFailure>(2);
 301                     failures.add(new ImportFailure(import_.location, import_.moduleName,
 302                             compilationResult == DoesNotExist.INSTANCE
 303                                     ? ImportFailure.MODULE_DOES_NOT_EXIST_REASON
 304                                     : ((Failure)compilationResult).errors));
 305                 }
 306             }
 307         }
 308
 309         if(failures != null && !robustly)
 310             throw new ImportFailureException(failures);
 311
 312         return result;
 313     }
 314
 315     private static THashMap<String, Module> mapEntriesToModules(THashMap<String, ModuleEntry> entries) {
 316         final THashMap<String, Module> result = new THashMap<String, Module>(entries.size());
 317         entries.forEachEntry(new TObjectObjectProcedure<String, ModuleEntry>() {
 318             @Override
 319             public boolean execute(String a, ModuleEntry b) {
 320                 result.put(a, b.compilationResult.getResult());
 321                 return true;
 322             }
 323         });
 324         return result;
 325     }
 326
 327     private static THashMap<String, RuntimeModule> mapEntriesToRuntimeModules(THashMap<String, ModuleEntry> entries) {
 328         final THashMap<String, RuntimeModule> result = new THashMap<String, RuntimeModule>(entries.size());
 329         entries.forEachEntry(new TObjectObjectProcedure<String, ModuleEntry>() {
 330             @Override
 331             public boolean execute(String a, ModuleEntry b) {
 332                 result.put(a, b.getRuntimeModule().getResult());
 333                 return true;
 334             }
 335         });
 336         return result;
 337     }
 338
 339     public Environment createEnvironment(
 340             ImportDeclaration[] imports,
 341             UpdateListener listener) throws ImportFailureException {
 342         return createEnvironment(null, imports, listener);
 343     }
 344
 345     public Environment createEnvironment(
 346             CompilationContext compilationContext,
 347             ImportDeclaration[] imports,
 348             UpdateListener listener) throws ImportFailureException {
 349         THashMap<String, ModuleEntry> entries = getModuleEntries(compilationContext, imports, listener, false);
 350         THashMap<String, Module> moduleMap = mapEntriesToModules(entries);
 351         return createEnvironment(moduleMap, imports);
 352     }
 353
 354     public Environment createEnvironmentRobustly(
 355             CompilationContext compilationContext,
 356             ImportDeclaration[] imports,
 357             UpdateListener listener) {
 358         try {
 359             THashMap<String, ModuleEntry> entries = getModuleEntries(compilationContext, imports, listener, true);
 360             THashMap<String, Module> moduleMap = mapEntriesToModules(entries);
 361             return createEnvironment(moduleMap, imports);
 362         } catch(ImportFailureException e) {
 363             // Should not happen because of robust flag
 364             return EmptyEnvironment.INSTANCE;
 365         }
 366     }
 367
 368     public Environment createEnvironment(
 369             EnvironmentSpecification specification,
 370             UpdateListener listener) throws ImportFailureException {
 371         return createEnvironment(specification.imports.toArray(new ImportDeclaration[specification.imports.size()]), listener);
 372     }
 373
 374     public RuntimeEnvironment createRuntimeEnvironment(
 375             EnvironmentSpecification environmentSpecification, ClassLoader parentClassLoader) throws ImportFailureException {
 376         return createRuntimeEnvironment(environmentSpecification, parentClassLoader, null);
 377     }
 378
 379     public RuntimeEnvironment createRuntimeEnvironment(EnvironmentSpecification environmentSpecification) throws ImportFailureException {
 380         return createRuntimeEnvironment(environmentSpecification, getClass().getClassLoader());
 381     }
 382
 383     public RuntimeEnvironment createRuntimeEnvironment(
 384             EnvironmentSpecification environmentSpecification,
 385             ClassLoader parentClassLoader,
 386             UpdateListener listener) throws ImportFailureException {
 387         return createRuntimeEnvironment(
 388                 environmentSpecification.imports.toArray(new ImportDeclaration[environmentSpecification.imports.size()]),
 389                 parentClassLoader,
 390                 listener);
 391     }
 392
 393     public RuntimeEnvironment createRuntimeEnvironment(
 394             ImportDeclaration[] imports,
 395             ClassLoader parentClassLoader,
 396             UpdateListener listener) throws ImportFailureException {
 397         THashMap<String, ModuleEntry> entries = getModuleEntries(null, imports, listener, false);
 398         THashMap<String, Module> moduleMap = mapEntriesToModules(entries);
 399         Environment environment = createEnvironment(moduleMap, imports);
 400         THashMap<String, RuntimeModule> runtimeModuleMap = mapEntriesToRuntimeModules(entries);
 401         RuntimeEnvironmentImpl result = new RuntimeEnvironmentImpl(environment, parentClassLoader, runtimeModuleMap);
 402         runtimeEnvironments.add(result);
 403         return result;
 404     }
 405
 406     private static Environment createEnvironment(
 407             THashMap<String, Module> moduleMap,
 408             ImportDeclaration[] imports) {
 409         NamespaceSpec spec = new NamespaceSpec();
 410         for(ImportDeclaration import_ : imports)
 411             if(import_.localName != null)
 412                 addToNamespace(moduleMap, spec, import_.moduleName, import_.localName,
 413                         NamespaceFilters.createFromSpec(import_.spec));
 414
 415         return new ConcreteEnvironment(moduleMap, spec.toNamespace());
 416     }
 417
 418     private static void addToNamespace(THashMap<String, Module> moduleMap,
 419             NamespaceSpec namespace, String moduleName, String localName,
 420             NamespaceFilter filter) {
 421         if(localName.isEmpty())
 422             addToNamespace(moduleMap, namespace, moduleName, filter);
 423         else
 424             addToNamespace(moduleMap, namespace.getNamespace(localName), moduleName, filter);
 425     }
 426
 427     private static void addToNamespace(THashMap<String, Module> moduleMap,
 428             NamespaceSpec namespace, String moduleName, NamespaceFilter filter) {
 429         ModuleImport moduleImport = namespace.moduleMap.get(moduleName);
 430         if(moduleImport == null) {
 431             Module module = moduleMap.get(moduleName);
 432             namespace.moduleMap.put(moduleName, new ModuleImport(module, filter));
 433             for(ImportDeclaration import_ : module.getDependencies())
 434                 if(import_.localName != null) {
 435                     NamespaceFilter localFilter = NamespaceFilters.createFromSpec(import_.spec);
 436                     if(import_.localName.equals(""))
 437                         localFilter = NamespaceFilters.intersection(filter, localFilter);
 438                     addToNamespace(moduleMap, namespace, import_.moduleName, import_.localName, localFilter);
 439                 }
 440         }
 441         else if(!filter.isSubsetOf(moduleImport.filter)) {
 442             moduleImport.filter = NamespaceFilters.union(moduleImport.filter, filter);
 443             for(ImportDeclaration import_ : moduleImport.module.getDependencies())
 444                 // We have to recheck only modules imported to this namespace
 445                 if("".equals(import_.localName)) {
 446                     NamespaceFilter localFilter = NamespaceFilters.createFromSpec(import_.spec);
 447                     localFilter = NamespaceFilters.intersection(filter, localFilter);
 448                     addToNamespace(moduleMap, namespace, import_.moduleName, import_.localName, localFilter);
 449                 }
 450         }
 451     }
 452
 453     public Object getValue(String moduleName, String valueName) throws ValueNotFound {
 454         Failable<RuntimeModule> module = getRuntimeModule(moduleName);
 455         if(module.didSucceed())
 456             return module.getResult().getValue(valueName);
 457         else if(module == DoesNotExist.INSTANCE)
 458             throw new ValueNotFound("Didn't find module " + moduleName);
 459         else
 460             throw new ValueNotFound(((Failure)module).toString());
 461     }
 462
 463     public Object getValue(String fullValueName) throws ValueNotFound {
 464         int p = fullValueName.lastIndexOf('/');
 465         if(p < 0)
 466             throw new ValueNotFound(fullValueName + " is not a valid full value name.");
 467         return getValue(fullValueName.substring(0, p), fullValueName.substring(p+1));
 468     }
 469
 470     public SCLValue getValueRef(String moduleName, String valueName) throws ValueNotFound {
 471         Failable<Module> module = getModule(moduleName);
 472         if(module.didSucceed()) {
 473             SCLValue value = module.getResult().getValue(valueName);
 474             if(value == null)
 475                 throw new ValueNotFound("Module " + moduleName + " does not contain value " + valueName + ".");
 476             return value;
 477         }
 478         else if(module == DoesNotExist.INSTANCE)
 479             throw new ValueNotFound("Didn't find module " + moduleName);
 480         else
 481             throw new ValueNotFound(((Failure)module).toString());
 482     }
 483
 484     public SCLValue getValueRef(String fullValueName) throws ValueNotFound {
 485         int p = fullValueName.lastIndexOf('/');
 486         if(p < 0)
 487             throw new ValueNotFound(fullValueName + " is not a valid full value name.");
 488         return getValueRef(fullValueName.substring(0, p), fullValueName.substring(p+1));
 489     }
 490
 491     public ModuleSourceRepository getSourceRepository() {
 492         return sourceRepository;
 493     }
 494
 495     public String getDocumentation(String documentationName) {
 496         String documentation = sourceRepository.getDocumentation(documentationName);
 497         if(documentation == null && parentRepository != null)
 498             return parentRepository.getDocumentation(documentationName);
 499         return documentation;
 500     }
 501
 502     /**
 503      * Flush clears module repository cache completely. It should not be called in
 504      * normal operation, but may be useful during testing for clearing repositories
 505      * that are statically defined.
 506      */
 507     public void flush() {
 508         if (parentRepository != null)
 509             parentRepository.flush();
 510         if (moduleCache != null)
 511             for (ModuleEntry entry : moduleCache.values())
 512                 entry.dispose();
 513         moduleCache = new ConcurrentHashMap<String, ModuleEntry>();
 514     }
 515
 516     /**
 517      * Gets the map of all modules that have been currently compiled successfully.
 518      * Not that the method does not return all possible modules in the source repository.
 519      */
 520     public Map<String, Module> getModules() {
 521         Map<String, Module> result = new HashMap<>(moduleCache.size());
 522         for (Map.Entry<String, ModuleEntry> entry : moduleCache.entrySet()) {
 523             ModuleEntry moduleEntry = entry.getValue();
 524             if (moduleEntry.compilationResult.didSucceed()) {
 525                 result.put(entry.getKey(), moduleEntry.compilationResult.getResult());
 526             }
 527         }
 528         return result;
 529     }
 530
 531     public ModuleCompilationOptionsAdvisor getAdvisor() {
 532         return advisor;
 533     }
 534
 535     public void setAdvisor(ModuleCompilationOptionsAdvisor advisor) {
 536         this.advisor = advisor;
 537     }
 538
 539     public Set<RuntimeEnvironment> getRuntimeEnvironments() {
 540         return runtimeEnvironments;
 541     }
 542
 543 }
 544