]> gerrit.simantics Code Review - simantics/platform.git/blob - bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/QuerySupportImpl.java
DB query swapping to file system
[simantics/platform.git] / bundles / org.simantics.db.procore / src / fi / vtt / simantics / procore / internal / QuerySupportImpl.java
1 package fi.vtt.simantics.procore.internal;
2
3 import java.io.ByteArrayInputStream;
4 import java.io.InputStream;
5 import java.util.Collection;
6 import java.util.function.Consumer;
7
8 import org.simantics.db.ObjectResourceIdMap;
9 import org.simantics.db.Resource;
10 import org.simantics.db.Session;
11 import org.simantics.db.Statement;
12 import org.simantics.db.VirtualGraph;
13 import org.simantics.db.WriteGraph;
14 import org.simantics.db.common.StandardStatement;
15 import org.simantics.db.common.request.WriteRequest;
16 import org.simantics.db.common.utils.Logger;
17 import org.simantics.db.exception.DatabaseException;
18 import org.simantics.db.exception.InvalidResourceReferenceException;
19 import org.simantics.db.exception.ResourceNotFoundException;
20 import org.simantics.db.impl.ClusterI;
21 import org.simantics.db.impl.ClusterSupport;
22 import org.simantics.db.impl.ForEachObjectContextProcedure;
23 import org.simantics.db.impl.ForEachObjectProcedure;
24 import org.simantics.db.impl.ResourceImpl;
25 import org.simantics.db.impl.TransientGraph;
26 import org.simantics.db.impl.VirtualGraphImpl;
27 import org.simantics.db.impl.graph.ReadGraphImpl;
28 import org.simantics.db.impl.query.IntProcedure;
29 import org.simantics.db.impl.query.QueryProcessor;
30 import org.simantics.db.impl.query.QuerySupport;
31 import org.simantics.db.impl.support.BuiltinSupport;
32 import org.simantics.db.impl.support.ResourceSupport;
33 import org.simantics.db.procore.cluster.ClusterImpl;
34 import org.simantics.db.procore.cluster.ClusterSmall;
35 import org.simantics.db.service.SerialisationSupport;
36 import org.simantics.utils.DataContainer;
37 import org.slf4j.LoggerFactory;
38
39 import gnu.trove.set.hash.TIntHashSet;
40
41 public class QuerySupportImpl implements QuerySupport {
42
43     private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(QuerySupportImpl.class);
44
45         final SessionImplSocket session;
46         final State state;
47         final ClusterTable clusterTable;
48         final BuiltinSupport builtinSupport;
49         final ClusterSupport clusterSupport;
50         final ResourceSupport resourceSupport;
51         final SerialisationSupport serializationSupport;
52         final VirtualGraphServerSupportImpl virtualGraphServerSupport;
53         final GraphSession graphSession;
54         final SessionRequestManager syncThreads;
55         final int root;
56
57     private boolean pendingPrimitives = false;
58         
59         QuerySupportImpl(SessionImplSocket session, ClusterSupport clusterSupport, SerialisationSupport serializationSupport, SessionRequestManager syncThreads) {
60                 this.session = session;
61                 this.state = session.state;
62                 this.clusterTable = session.clusterTable;
63                 this.resourceSupport = session.resourceSupport;
64                 this.virtualGraphServerSupport = session.virtualGraphServerSupport;
65                 this.graphSession = session.graphSession;
66                 this.builtinSupport = session.builtinSupport;
67                 this.clusterSupport = clusterSupport;
68                 this.serializationSupport = serializationSupport;
69                 this.syncThreads = syncThreads;
70                 this.root = getBuiltin("http:/");
71                 assert(this.session != null);
72                 assert(this.state != null);
73                 assert(this.clusterTable != null);
74                 assert(this.resourceSupport != null);
75                 assert(this.virtualGraphServerSupport != null);
76                 assert(this.graphSession != null);
77                 assert(this.builtinSupport != null);
78                 assert(this.clusterSupport != null);
79                 assert(this.serializationSupport != null);
80                 assert(this.syncThreads != null);
81         }
82
83         @Override
84         public ResourceSupport getSupport() {
85                 return resourceSupport;
86         }
87
88     @Override
89     public ClusterSupport getClusterSupport() {
90         return clusterSupport;
91     }
92
93     @Override
94     public ObjectResourceIdMap<String> createChildMap() {
95         return new ObjectResourceMap<String>(session);
96     }
97
98     @Override
99     public Statement getStatement(int s, int p, int o) {
100         return getStatement(null, s, p, o);
101     }
102
103     @Override
104     public Statement getStatement(ReadGraphImpl graph, int s, int p, int o) {
105         Resource sr = getResource(s);
106         Resource pr = getResource(p);
107         Resource or = getResource(o);
108         return new StandardStatement(sr, pr, or);
109     }
110
111         @Override
112         public Session getSession() {
113                 return session;
114         }
115
116     @Override
117     public long getClusterId(int id) {
118         ClusterI cluster = clusterTable.getClusterByResourceKey(id);
119         if(cluster == null) return 0;
120         return cluster.getClusterId();
121     }
122     
123     @Override
124     public boolean isImmutable(int id) {
125         // Virtuals are mutable
126         if(id < 0) return false;
127         // Root library is mutable
128         if(root == id) return false;
129         // Anything goes in service mode
130         if(session.serviceMode > 0) return false;
131         return clusterTable.isImmutable(id);
132     }
133
134     @Override
135     public int getId(Resource resource) {
136         if (resource instanceof ResourceImpl)
137             return ((ResourceImpl)resource).id;
138         return 0;
139     }
140
141         @Override
142         public Resource getResource(int id) {
143                 try {
144                         return serializationSupport.getResource(id);
145                 } catch (DatabaseException e) {
146                         // TODO: consider changing this method to throw something
147                         LOGGER.error("getResource(" + id + ") failed", e);
148                 }
149                 return null;
150         }
151
152         @Override
153     public boolean resume(ReadGraphImpl graph) {
154                 
155                 return syncThreads.session.queryProvider2.resume(graph);
156                 
157     }
158
159 //    @Override
160 //    final public void sync(int resumeThread, final SessionRunnable runnable) {
161 //      
162 //      syncThreads.session.queryProvider2.schedule(Integer.MIN_VALUE, new SessionTask(resumeThread) {
163 //
164 //                      @Override
165 //                      public void run(int thread) {
166 //                              runnable.run(thread);
167 //                      }
168 //              
169 //      });
170 //      
171 //    }
172
173 //    @Override
174 //    final public int nextSyncThread() {
175 //      throw new Error();
176 ////        return syncThreads.nextThread();
177 //    }
178
179         @Override
180     public void dirtyPrimitives() {
181         session.dirtyPrimitives = true;
182         if(state.getWriteCount() == 0 && !pendingPrimitives) {
183             pendingPrimitives = true;
184             session.asyncRequest(new WriteRequest() {
185
186                 @Override
187                 public void perform(WriteGraph graph) throws DatabaseException {
188                     pendingPrimitives = false;
189                 }
190                 
191             });
192         }
193     }
194
195     @Override
196     @Deprecated
197     final public void aboutToRead() {
198     }
199
200 //    @Override
201 //    public void increaseReferenceCount(int callerThread, int subject) {
202 //        if (subject < 0)
203 //            return;
204 //        ClusterI proxy = clusterTable.getClusterByResourceKey(subject);
205 //        if (null == proxy)
206 //            return;
207 //        proxy.increaseReferenceCount(callerThread, 1);
208 //    }
209 //
210 //    @Override
211 //    public void decreaseReferenceCount(int callerThread, int subject) {
212 //        if (subject < 0)
213 //            return;
214 //        ClusterProxy proxy = clusterTable.getClusterByResourceKey(subject);
215 //        if (null == proxy)
216 //            return;
217 //        proxy.decreaseReferenceCount(callerThread, 1);
218 //    }
219
220 //      @Override
221         public void getObjects4(final ReadGraphImpl graph, final int subject, final ForEachObjectProcedure procedure) {
222                 
223                 if(subject < 0) {
224                         
225                         for(TransientGraph g : virtualGraphServerSupport.providers) {
226                 for (final int id : g.getObjects(subject, procedure.predicateKey)) {
227
228 //                    int suggestSchedule = graph.processor.processor.resourceThread(id);
229 //                    if(graph.callerThread == suggestSchedule) {
230                         try {
231                                                         procedure.execute(graph, new ResourceImpl(resourceSupport, id));
232                                                 } catch (DatabaseException e) {
233                                                         LOGGER.error("Unexpected exception while handling object", e);
234                                                 }
235 //                    } else {
236 //                      graph.processor.processor.schedule(graph.callerThread, new SessionTask(suggestSchedule) {
237 //              
238 //                              @Override
239 //                              public void run(int thread) {
240 //                              procedure.execute(graph.newAsync(thread), new ResourceImpl(resourceSupport, id));
241 //                              }
242 //              
243 //                      });
244 //                    }
245                         
246                 }
247                         }
248                 try {
249                         procedure.finished(graph);
250                         } catch (DatabaseException e) {
251                                 LOGGER.error("Unexpected exception while handling objects", e);
252                         }
253 //              graph.dec();
254                 return;
255                 
256                 } 
257                 
258         final ClusterImpl cluster = (ClusterImpl)clusterTable.getClusterByResourceKey(subject);
259         if(!cluster.isLoaded()) {
260                 try {
261                                 cluster.load();
262                         } catch (DatabaseException e) {
263                                 LOGGER.error("Unexpected exception while handling objects", e);
264                         }
265                 getObjects4(graph, subject, procedure);
266                 return;
267         }
268                 
269                 if (cluster.hasVirtual() && virtualGraphServerSupport.virtuals.contains(subject)) {
270                         
271                         for(TransientGraph g : virtualGraphServerSupport.providers) {
272                 for (final int id : g.getObjects(subject, procedure.predicateKey)) {
273                         try {
274                         procedure.execute(graph, new ResourceImpl(resourceSupport, id));
275                                 } catch (DatabaseException e) {
276                                         LOGGER.error("Unexpected exception while handling objects", e);
277                                 }
278                         
279                         
280 //                    int suggestSchedule = graph.processor.processor.resourceThread(id);
281 //                    if(graph.callerThread == suggestSchedule) {
282 //                    } else {
283 //                      graph.processor.processor.schedule(graph.callerThread, new SessionTask(suggestSchedule) {
284 //              
285 //                              @Override
286 //                              public void run(int thread) {
287 //                              procedure.execute(graph.newAsync(thread), new ResourceImpl(resourceSupport, id));
288 //                              }
289 //              
290 //                      });
291 //                    }
292                 }
293                         }
294                         
295                         try {
296                                 cluster.forObjects(graph, subject, procedure);
297                         } catch (DatabaseException e) {
298                                 e.printStackTrace();
299                         }
300                         
301                 } else {
302                         
303                         try {
304                                 cluster.forObjects(graph, subject, procedure);
305                         } catch (DatabaseException e) {
306                                 e.printStackTrace();
307                         }
308                         
309                 }
310                 
311         }
312
313         public <C> void getObjects4(final ReadGraphImpl graph, final int subject, final C context, final ForEachObjectContextProcedure<C> procedure) {
314                 
315                 if(subject < 0) {
316                         
317                         for(TransientGraph g : virtualGraphServerSupport.providers) {
318                 for (final int id : g.getObjects(subject, procedure.predicateKey)) {
319
320 //                    int suggestSchedule = graph.processor.processor.resourceThread(id);
321 //                    if(graph.callerThread == suggestSchedule) {
322                         
323                         try {
324                                 procedure.execute(graph, context, new ResourceImpl(resourceSupport, id));
325                                 } catch (DatabaseException e) {
326                                         LOGGER.error("Unexpected exception while handling objects", e);
327                                 }
328                         
329                         
330                         
331 //                    } else {
332 //                      graph.processor.processor.schedule(graph.callerThread, new SessionTask(suggestSchedule) {
333 //              
334 //                              @Override
335 //                              public void run(int thread) {
336 //                              procedure.execute(graph.newAsync(thread), context, new ResourceImpl(resourceSupport, id));
337 //                              }
338 //              
339 //                      });
340 //                    }
341                         
342                 }
343                         }
344                         
345                 try {
346                         procedure.finished(graph, context);
347                         } catch (DatabaseException e) {
348                                 LOGGER.error("Unexpected exception while handling objects", e);
349                         }
350
351                         
352 //              graph.dec();
353                 return;
354                 
355                 } 
356                 
357         final ClusterImpl cluster = (ClusterImpl)clusterTable.getClusterByResourceKey(subject);
358         if(!cluster.isLoaded()) {
359                 cluster.load(session.clusterTranslator, new Runnable() {
360
361                                 @Override
362                                 public void run() {
363                                         getObjects4(graph, subject, context, procedure);
364                                 }
365                         
366                 });
367                 return;
368         }
369                 
370                 if (cluster.hasVirtual() && virtualGraphServerSupport.virtuals.contains(subject)) {
371                         
372                         for(TransientGraph g : virtualGraphServerSupport.providers) {
373                 for (final int id : g.getObjects(subject, procedure.predicateKey)) {
374 //                    int suggestSchedule = graph.processor.processor.resourceThread(id);
375 //                    if(graph.callerThread == suggestSchedule) {
376                     try {
377                         procedure.execute(graph, context, new ResourceImpl(resourceSupport, id));
378                     } catch (DatabaseException e) {
379                         LOGGER.error("Unexpected exception while handling objects", e);
380                     }
381 //                    } else {
382 //                      graph.processor.processor.schedule(graph.callerThread, new SessionTask(suggestSchedule) {
383 //              
384 //                              @Override
385 //                              public void run(int thread) {
386 //                              procedure.execute(graph.newAsync(thread), context, new ResourceImpl(resourceSupport, id));
387 //                              }
388 //              
389 //                      });
390 //                    }
391                 }
392                         }
393                         
394                         try {
395                                 cluster.forObjects(graph, subject, context, procedure);
396                         } catch (DatabaseException e) {
397                                 e.printStackTrace();
398                         }
399                         
400                 } else {
401                         
402                         try {
403                                 cluster.forObjects(graph, subject, context, procedure);
404                         } catch (DatabaseException e) {
405                                 e.printStackTrace();
406                         }
407                         
408                 }
409                 
410         }
411         
412         @Override
413     public int getSingleInstance(final int subject) {
414         
415         // Do not process this information for virtual resources
416         if(subject < 0) return 0;
417         
418         final ClusterI cluster = clusterTable.getClusterByResourceKey(subject);
419         if (cluster == null)
420             System.out.println("null cluster: " + Integer.toString(subject, 16));
421         assert (cluster != null);
422
423         try {
424         
425                 ClusterI.CompleteTypeEnum type = cluster.getCompleteType(subject, clusterSupport);
426                 if(ClusterI.CompleteTypeEnum.InstanceOf == type) {
427                         int result = cluster.getCompleteObjectKey(subject, clusterSupport);
428                         assert(result > 0);
429                         return result;
430                 } else {
431                         return 0;
432                 }
433         
434         } catch (DatabaseException e) {
435                 
436                 Logger.defaultLogError(e);
437                 return 0;
438                 
439         }
440         // This happens is the resource is bogus
441         catch (Throwable t) {
442                 
443                 analyseProblem(cluster);
444                 
445                 Logger.defaultLogError(t);
446                 
447                 return 0;
448                 
449         }
450         
451     }
452
453         @Override
454     public int getSingleSuperrelation(final int subject) {
455         
456         // Do not process this information for virtual resources
457         if(subject < 0) return 0;
458
459         final ClusterI cluster = clusterTable.getClusterByResourceKey(subject);
460         if (cluster == null)
461             System.out.println("null cluster: " + Integer.toString(subject, 16));
462         assert (cluster != null);
463
464         try {
465         
466                 ClusterI.CompleteTypeEnum type = cluster.getCompleteType(subject, clusterSupport);
467                 if(ClusterI.CompleteTypeEnum.SubrelationOf == type) {
468                         int result = cluster.getCompleteObjectKey(subject, clusterSupport);
469                         assert(result > 0);
470                         return result;
471                 } else {
472                         return 0;
473                 }
474         
475         } catch (DatabaseException e) {
476                 
477                 Logger.defaultLogError(e);
478                 return 0;
479                 
480         }
481         
482     }
483
484         @Override
485     public boolean getObjects(final ReadGraphImpl graph, final int subject, final int predicate, final IntProcedure procedure) throws DatabaseException {
486                 
487                 assert (subject != 0);
488                 assert (predicate != 0);
489
490                 if(subject < 0) {
491
492                         boolean found = false;
493                         
494                         for(TransientGraph g : virtualGraphServerSupport.providers) {
495                                 for (final int id : g.getObjects(subject, predicate)) {
496                                         found = true;
497                                         procedure.execute(graph, id);
498                                 }
499                         }
500                         
501                         return found;
502
503                 }
504
505                 final ClusterI cluster = clusterTable.getClusterByResourceKey(subject);
506                 assert(cluster.isLoaded());
507         
508         if (cluster.hasVirtual() && virtualGraphServerSupport.virtuals.contains(subject)) {
509                 
510                 final DataContainer<Boolean> found = new DataContainer<Boolean>(Boolean.FALSE);
511                 final TIntHashSet result = new TIntHashSet(5);
512                 
513                         for(TransientGraph g : virtualGraphServerSupport.providers) {
514                                 for (final int id : g.getObjects(subject, predicate)) {
515
516                                         found.set(true);
517                                         if (result.add(id)) {
518                                                 procedure.execute(graph, id);
519                                         }
520
521                                 }
522                         }
523                         
524                         // Virtual predicates are not found from persistent clusters
525                         if(predicate < 0) return found.get();
526
527                 // wheels within wheels
528                 final ClusterI.ObjectProcedure<Object> proc = new ClusterI.ObjectProcedure<Object>() {
529
530                         @Override
531                         public boolean execute(Object context, int object) {
532
533                                 found.set(true);
534                                 if (result.add(object)) {
535                                         try {
536                                                         procedure.execute(graph, object);
537                                                 } catch (DatabaseException e) {
538                                                         Logger.defaultLogError(e);
539                                                 }
540                                 }
541
542                                 return false; // continue looping
543
544                         }
545
546                 };
547                         
548                         try {
549                 cluster.forObjects(subject, predicate, proc, null, clusterSupport);
550                         } catch (DatabaseException e) {
551                                 e.printStackTrace();
552                         }
553                         
554                         return found.get();
555                         
556                 } else {
557                         
558                         // Virtual predicates are not found from persistent clusters
559                         if(predicate < 0) return false;
560                         
561                         class A implements ClusterI.ObjectProcedure<Object> {
562
563                         boolean found = false;
564
565                         @Override
566                         public boolean execute(Object context, int object) {
567
568                                         found = true;
569                                         try {
570                                                 procedure.execute(graph, object);
571                                         } catch (DatabaseException e) {
572                                                 Logger.defaultLogError(e);
573                                         }
574                                         return false; // continue looping
575
576                                 }
577
578                                 public boolean found() {
579                                         return found;
580                                 }
581
582                         }
583                         
584                 // wheels within wheels
585                 final A proc = new A();
586                 
587                         try {
588                 cluster.forObjects(subject, predicate, proc, null, clusterSupport);
589                         } catch (DatabaseException e) {
590                                 Logger.defaultLogError(e);
591                         } 
592                         // This happens if resource is bogus
593                         catch (Throwable t) {
594                                 
595                         analyseProblem(cluster);
596                         
597                         Logger.defaultLogError(t);
598                                 
599                         }
600                 
601                 return proc.found();
602                         
603                 }
604         
605     }
606
607 //      @Override
608 //    public boolean getFunctionalObject(final ReadGraphImpl graph, final int subject, final int predicate,
609 //            final IntProcedure procedure) {
610 //
611 //        assert (subject != 0);
612 //        assert (predicate != 0);
613 //
614 //        if(subject < 0) {
615 //
616 //              boolean found = false;
617 //
618 //              Collection<TransientGraph> providers = virtualGraphServerSupport.getVirtualGraphs(subject);
619 //              if (providers != null) {
620 //
621 //                      for (VirtualGraph provider : providers) {
622 //
623 //                              for (int id : ((VirtualGraphImpl)provider).getObjects(subject, predicate)) {
624 //                                      found = true;
625 //                      procedure.execute(graph, id);
626 //                    }
627 //
628 //                }
629 //
630 //            }
631 //              
632 //              return found;
633 //              
634 //        }
635 //        
636 //        final ClusterI cluster = clusterTable.getClusterByResourceKey(subject);
637 //        
638 //        if (cluster.hasVirtual() && virtualGraphServerSupport.virtuals.contains(subject)) {
639 //        
640 //              final DataContainer<Boolean> found = new DataContainer<Boolean>(false);
641 //              final TIntHashSet result = new TIntHashSet(5);
642 //
643 //              Collection<TransientGraph> providers = virtualGraphServerSupport.getVirtualGraphs(subject);
644 //              if (providers != null) {
645 //
646 //                      for (VirtualGraph provider : providers) {
647 //
648 //                              for (int id : ((VirtualGraphImpl)provider).getObjects(subject, predicate)) {
649 //                                      found.set(true);
650 //                      procedure.execute(graph, id);
651 //                    }
652 //
653 //                }
654 //
655 //            }
656 //              
657 //            // wheels within wheels
658 //            final ClusterI.ObjectProcedure<Object> proc = new ClusterI.ObjectProcedure<Object>() {
659 //
660 //                @Override
661 //                public boolean execute(int callerThread, Object context, int object) {
662 //
663 //                    found.set(true);
664 //                    System.out.println("-found object " + object);
665 //                    if (result.add(object)) {
666 //                      procedure.execute(graph.newAsync(callerThread), object);
667 //                    }
668 //
669 //                    return false; // continue looping
670 //
671 //                }
672 //                
673 //                              @Override
674 //                              public boolean found() {
675 //                                      throw new UnsupportedOperationException();
676 //                              }
677 //                
678 //            };
679 //            
680 //            try {
681 //                cluster.forObjects(graph.callerThread, subject, predicate, proc, null, clusterSupport);
682 //            } catch (DatabaseException e) {
683 //                Logger.defaultLogError(e);
684 //                return false;
685 //            } catch (Throwable t) {
686 //                Logger.defaultLogError(t);
687 //              t.printStackTrace();
688 //            }
689 //            
690 //            return found.get();
691 //            
692 //        } else {
693 //
694 //              // wheels within wheels
695 //              final ClusterI.ObjectProcedure<Object> proc = new ClusterI.ObjectProcedure<Object>() {
696 //                  
697 //                      boolean found = false;
698 //                      
699 //                  @Override
700 //                  public boolean execute(int callerThread, Object context, int object) {
701 //      
702 //                      found = true;
703 //                      procedure.execute(graph.newAsync(callerThread), object);
704 //                      return false; // continue looping
705 //                      
706 //                  }
707 //                  
708 //                              @Override
709 //                              public boolean found() {
710 //                                      return found;
711 //                              }
712 //                  
713 //                  @Override
714 //                  public String toString() {
715 //                      return "Wheels for " + procedure;
716 //                  }
717 //                  
718 //              };
719 //              
720 //              try {
721 //                  cluster.forObjects(graph.callerThread, subject, predicate, proc, null, clusterSupport);
722 //              } catch (DatabaseException e) {
723 //                  Logger.defaultLogError(e);
724 //                  return false;
725 //              } catch (Throwable t) {
726 //                      t.printStackTrace();
727 //                  Logger.defaultLogError(t);
728 //                  return false;
729 //              }
730 //              
731 //              return proc.found();
732 //        
733 //        }
734 //        
735 //    }
736         
737         @Override
738     public int getFunctionalObject(final int subject, final int predicate) {
739
740         assert (subject != 0);
741         assert (predicate != 0);
742
743         if(subject < 0) {
744
745                 int found = 0;
746
747                 Collection<TransientGraph> providers = virtualGraphServerSupport.getVirtualGraphs(subject);
748                 if (providers != null) {
749
750                         for (VirtualGraph provider : providers) {
751
752                                 for (int id : ((VirtualGraphImpl)provider).getObjects(subject, predicate)) {
753                                         if(found == 0) found = id;
754                                         else found = -1;
755                     }
756
757                 }
758
759             }
760                 
761                 return found;
762 //              if(found == -1) return 0;
763 //              else return found;
764                 
765         }
766         
767         final ClusterI cluster = clusterTable.getClusterByResourceKey(subject);
768         
769         if (cluster.hasVirtual() && virtualGraphServerSupport.virtuals.contains(subject)) {
770
771                 int result = 0;
772                 Collection<TransientGraph> providers = virtualGraphServerSupport.getVirtualGraphs(subject);
773                 if (providers != null) {
774
775                         for (VirtualGraph provider : providers) {
776
777                                 for (int id : ((VirtualGraphImpl)provider).getObjects(subject, predicate)) {
778                                         if(result == 0) result = id;
779                                         else result = -1;
780                     }
781
782                 }
783
784             }
785                 
786                 if(result != 0) return result; 
787             
788                 try {
789                                 return cluster.getSingleObject(subject, predicate, clusterSupport);
790                         } catch (DatabaseException e) {
791                                 return -1;
792                         }
793             
794         } else {
795
796                 try {
797                                 return cluster.getSingleObject(subject, predicate, clusterSupport);
798                         } catch (DatabaseException e) {
799                                 return -1;
800                         }
801                 // This comes if the resource is bogus
802                 catch (Throwable t) {
803                         
804                 analyseProblem(cluster);
805                 
806                 Logger.defaultLogError(t);
807                         
808                                 return -1;
809                                 
810                         }
811         
812         }
813         
814     }
815
816 //      @Override
817 //    public boolean getStatements(final ReadGraphImpl graph, final int subject, final int predicate,
818 //            final TripleIntProcedure procedure, final Statements entry) {
819 //
820 //        assert (subject != 0);
821 //        assert (predicate != 0);
822 //
823 //        final TIntHashSet result = new TIntHashSet(16);
824 //        final DataContainer<Boolean> found = new DataContainer<Boolean>(false);
825 //
826 //        Collection<TransientGraph> providers = virtualGraphServerSupport.getVirtualGraphs(subject);
827 //        
828 //        if (providers != null) {
829 //
830 //            for (VirtualGraph provider : providers) {
831 //
832 //                for (int id : ((VirtualGraphImpl)provider).getObjects(subject, predicate)) {
833 //
834 //                    found.set(true);
835 //
836 //                    if (result.add(id)) {
837 //                        if (null != entry) {
838 //                            entry.addOrSet(subject, predicate, id);
839 //                            procedure.execute(graph, subject, predicate, id);
840 //                            return true; // break;
841 //                        } else {
842 //                            procedure.execute(graph, subject, predicate, id);
843 //                        }
844 //                    }
845 //
846 //                }
847 //
848 //            }
849 //
850 //        }
851 //
852 //        if (subject < 0)
853 //            return found.get();
854 //
855 //        final ClusterI cluster = clusterTable.getClusterByResourceKey(subject);
856 //        assert (cluster != null);
857 //
858 //        // wheels within wheels
859 //        final ClusterI.ObjectProcedure<Object> proc = new ClusterI.ObjectProcedure<Object>() {
860 //            @Override
861 //            public boolean execute(int callerThread, Object context, int object) {
862 //
863 //                found.set(true);
864 //
865 //                if (result.add(object)) {
866 //                    if (null != entry) {
867 //                        entry.addOrSet(subject, predicate, object);
868 //                        procedure.execute(graph.newAsync(callerThread), subject, predicate, object);
869 //                        return true; // break;
870 //                    } else {
871 //                        procedure.execute(graph.newAsync(callerThread), subject, predicate, object);
872 //                    }
873 //                }
874 //
875 //                return false; // continue looping
876 //
877 //            }
878 //            
879 //                      @Override
880 //                      public boolean found() {
881 //                              throw new UnsupportedOperationException();
882 //                      }
883 //            
884 //        };
885 //
886 //        try {
887 //            cluster.forObjects(graph.callerThread, subject, predicate, proc, null, clusterSupport);
888 //        } catch (DatabaseException e) {
889 //            Logger.defaultLogError(e);
890 //        }
891 //        return found.get();
892 //
893 //    }
894
895         @Override
896     public org.simantics.db.DirectStatements getStatements(final ReadGraphImpl graph, final int subject, final QueryProcessor processor, boolean ignoreVirtual) {
897
898         assert (subject != 0);
899
900         final DirectStatementsImpl result = new DirectStatementsImpl(resourceSupport, subject);
901
902         if (!ignoreVirtual) {
903             Collection<TransientGraph> providers = virtualGraphServerSupport.getVirtualGraphs(subject);
904             if (providers != null) {
905                 for (TransientGraph provider : providers) {
906                     for (int p : provider.getPredicates(subject)) {
907                         for (int o : provider.getObjects(subject, p)) {
908                             result.addStatement(p, o);
909                         }
910                     }
911                 }
912             }
913         }
914
915         if (subject < 0) {
916                 return result;
917         } else {
918             final ClusterI cluster = clusterTable.getClusterByResourceKey(subject);
919             assert (cluster != null);
920             doGetStatements(graph, cluster, subject, result);
921         }
922         
923         return result;
924
925     }
926
927     @Override
928     public void getPredicates(final ReadGraphImpl graph, final int subject, final IntProcedure procedure) throws DatabaseException {
929
930         final TIntHashSet result = new TIntHashSet(16);
931
932         Collection<TransientGraph> providers = virtualGraphServerSupport.getVirtualGraphs(subject);
933         
934         if (providers != null) {
935
936             for (VirtualGraph provider : providers) {
937
938                 for (int id : ((VirtualGraphImpl)provider).getPredicates(subject)) {
939
940                     if (result.add(id)) {
941                         procedure.execute(graph, id);
942                     }
943
944                 }
945
946             }
947
948         }
949
950         if (subject < 0) {
951             procedure.finished(graph);
952             return;
953         }
954
955         ClusterI proxy = clusterTable.getClusterByResourceKey(subject);
956         assert (proxy != null);
957
958         final DataContainer<Integer> got = new DataContainer<Integer>(0);
959         ClusterI.PredicateProcedure<Object> proc = new ClusterI.PredicateProcedure<Object>() {
960             @Override
961             public boolean execute(Object context, int predicate, int oi) {
962                 if (result.add(predicate)) {
963                     try {
964                         procedure.execute(graph, predicate);
965                     } catch (DatabaseException e) {
966                         Logger.defaultLogError(e);
967                     }
968                 }
969                 got.set(got.get() + 1);
970                 return false; // continue looping
971             }
972         };
973         try {
974             proxy.forPredicates(subject, proc, null, clusterSupport);
975         } catch (DatabaseException e) {
976             Logger.defaultLogError(e);
977         }
978         procedure.finished(graph);
979
980     }
981     
982     
983
984 //    @Override
985 //    public void getValue(ReadGraphImpl graph, int resource, InternalProcedure<byte[]> procedure) {
986 //
987 //      if(resource < 0) {
988 //      
989 //              Collection<TransientGraph> providers = virtualGraphServerSupport.getVirtualGraphs(resource);
990 //              
991 //              if (providers != null) {
992 //      
993 //                  for (VirtualGraph provider : providers) {
994 //      
995 //                      Object value = ((VirtualGraphImpl)provider).getValue(resource);
996 //                      if (value != null) {
997 //                          procedure.execute(graph, (byte[])value);
998 //                          return;
999 //                      }
1000 //      
1001 //                  }
1002 //      
1003 //              }
1004 //              
1005 //              return;
1006 //        
1007 //      }
1008 //      
1009 //      ClusterI cluster = clusterTable.getClusterByResourceKey(resource);
1010 //        if (cluster.hasVirtual() && virtualGraphServerSupport.virtuals.contains(resource)) {
1011 //              
1012 //              Collection<TransientGraph> providers = virtualGraphServerSupport.getVirtualGraphs(resource);
1013 //              
1014 //              if (providers != null) {
1015 //      
1016 //                  for (VirtualGraph provider : providers) {
1017 //      
1018 //                      Object value = ((VirtualGraphImpl)provider).getValue(resource);
1019 //                      if (value != null) {
1020 //                          procedure.execute(graph, (byte[])value);
1021 //                          return;
1022 //                      }
1023 //      
1024 //                  }
1025 //      
1026 //              }
1027 //              
1028 //              try {
1029 //                      
1030 //                      byte[] data = cluster.getValue(resource, clusterSupport);
1031 //                      if (null == data || 0 == data.length) {
1032 //                          procedure.execute(graph, null);
1033 //                      } else {
1034 //                          procedure.execute(graph, data);
1035 //                      }
1036 //                      
1037 //              } catch (DatabaseException e) {
1038 //                  Logger.defaultLogError(e);
1039 //              }
1040 //              
1041 //        } else {
1042 //
1043 //              try {
1044 //                      
1045 //                      byte[] data = cluster.getValue(resource, clusterSupport);
1046 //                      if (null == data || 0 == data.length) {
1047 //                          procedure.execute(graph, null);
1048 //                      } else {
1049 //                          procedure.execute(graph, data);
1050 //                      }
1051 //                      
1052 //              } catch (DatabaseException e) {
1053 //                  Logger.defaultLogError(e);
1054 //              }
1055 //
1056 //        }
1057 //
1058 //    }
1059
1060
1061     @Override
1062     public byte[] getValue(ReadGraphImpl graph, int resource) {
1063
1064         if(resource < 0) {
1065         
1066                 Collection<TransientGraph> providers = virtualGraphServerSupport.getVirtualGraphs(resource);
1067                 
1068                 if (providers != null) {
1069         
1070                     for (VirtualGraph provider : providers) {
1071         
1072                         Object value = ((VirtualGraphImpl)provider).getValue(resource);
1073                         if (value != null) {
1074                                 return (byte[])value;
1075                         }
1076         
1077                     }
1078         
1079                 }
1080                 
1081                 return null;
1082         
1083         }
1084         
1085         ClusterI cluster = clusterTable.getClusterByResourceKey(resource);
1086         if (cluster.hasVirtual() && virtualGraphServerSupport.virtuals.contains(resource)) {
1087                 
1088                 Collection<TransientGraph> providers = virtualGraphServerSupport.getVirtualGraphs(resource);
1089                 
1090                 if (providers != null) {
1091         
1092                     for (VirtualGraph provider : providers) {
1093         
1094                         Object value = ((VirtualGraphImpl)provider).getValue(resource);
1095                         if (value != null) {
1096                                 return (byte[])value;
1097                         }
1098         
1099                     }
1100         
1101                 }
1102                 
1103                 try {
1104                         
1105                         byte[] data = cluster.getValue(resource, clusterSupport);
1106                         if (null != data && 0 != data.length) {
1107                                 return data;
1108                         }
1109                         
1110                 } catch (DatabaseException e) {
1111                     Logger.defaultLogError(e);
1112                 }
1113                 
1114                 return null;
1115                 
1116         } else {
1117
1118                 try {
1119                         
1120                         byte[] data = cluster.getValue(resource, clusterSupport);
1121                         if (null != data && 0 != data.length) {
1122                                 return data;
1123                         }
1124                         
1125                 } catch (DatabaseException e) {
1126                     Logger.defaultLogError(e);
1127                 }
1128                 
1129                 return null;
1130
1131         }
1132
1133     }
1134
1135     @Override
1136     public InputStream getValueStream(ReadGraphImpl graph, int resource) {
1137
1138         if(resource < 0) {
1139         
1140             Collection<TransientGraph> providers = virtualGraphServerSupport.getVirtualGraphs(resource);
1141             
1142             if (providers != null) {
1143     
1144                 for (VirtualGraph provider : providers) {
1145     
1146                     Object value = ((VirtualGraphImpl)provider).getValue(resource);
1147                     if (value != null) {
1148                         return new ByteArrayInputStream((byte[])value);
1149                     }
1150     
1151                 }
1152     
1153             }
1154             
1155             return null;
1156         
1157         }
1158         
1159         ClusterI cluster = clusterTable.getClusterByResourceKey(resource);
1160         if (cluster.hasVirtual() && virtualGraphServerSupport.virtuals.contains(resource)) {
1161             
1162             Collection<TransientGraph> providers = virtualGraphServerSupport.getVirtualGraphs(resource);
1163             
1164             if (providers != null) {
1165     
1166                 for (VirtualGraph provider : providers) {
1167     
1168                     Object value = ((VirtualGraphImpl)provider).getValue(resource);
1169                     if (value != null) {
1170                         return new ByteArrayInputStream((byte[])value);
1171                     }
1172     
1173                 }
1174     
1175             }
1176             
1177             try {
1178                 
1179                 return cluster.getValueStream(resource, clusterSupport);
1180                 
1181             } catch (DatabaseException e) {
1182                 Logger.defaultLogError(e);
1183             }
1184             
1185             return null;
1186             
1187         } else {
1188
1189             try {
1190                 
1191                 return cluster.getValueStream(resource, clusterSupport);
1192                 
1193             } catch (DatabaseException e) {
1194                 Logger.defaultLogError(e);
1195             }
1196             
1197             return null;
1198
1199         }
1200
1201     }
1202     
1203     @Override
1204     public void requestCluster(ReadGraphImpl graph, final long clusterId, final Runnable r) {
1205
1206         class CallbackAdapter implements Consumer<DatabaseException> {
1207                 final Runnable r;
1208                 CallbackAdapter(final Runnable r) {
1209                         this.r = r;
1210                 }
1211                 @Override
1212                 public void accept(DatabaseException e) {
1213                         if (null != e)
1214                                 e.printStackTrace();
1215                         else
1216                                 r.run();
1217                 }
1218                 
1219         }
1220         
1221         double p = clusterTable.getLoadProbability();
1222 // System.out.print("Load cluster " + clusterId + " with probability " + p +
1223         // " -> ");
1224         final ClusterI proxy = clusterSupport.getClusterByClusterId(clusterId);
1225         if (!proxy.isLoaded()) {
1226                 clusterTable.gc();
1227             if (Math.random() < p) {
1228                 proxy.load(new CallbackAdapter(r));
1229             } else {
1230                 r.run();
1231             }
1232         } else {
1233             r.run();
1234         }
1235
1236     }
1237
1238     @Override
1239     public int getBuiltin(String uri) {
1240         return builtinSupport.getBuiltin(uri);
1241     }
1242
1243     @Override
1244     public void checkTasks() {
1245         System.out.println(syncThreads.toString());
1246     }
1247
1248 //    @Override
1249 //    public void asyncWrite(Write request) {
1250 //        
1251 ////            if(plainWrite(writer) && sameProvider(request)) {
1252 ////                    writer.writeSupport.pushRequest(request);
1253 ////            } else {
1254 //              asyncRequest(request);
1255 ////            }
1256 //        
1257 //    }
1258
1259     /*
1260      * Helpers
1261      * 
1262      * 
1263      */
1264     
1265     private void doGetStatements(ReadGraphImpl graph, final ClusterI cluster, final int subject, final DirectStatementsImpl result) {
1266
1267         final class Proc implements ClusterI.PredicateProcedure<Object> {
1268
1269                         @Override
1270                         public boolean execute(Object context, final int predicate, final int objectIndex) {
1271                                 
1272                                 doExecute(null, predicate, objectIndex);
1273                                 return false;
1274                                 
1275                         }
1276
1277                         private void doExecute(Object context, final int predicate, final int objectIndex) {
1278
1279                                 try {
1280                                         cluster.forObjects(subject, predicate, new ClusterI.ObjectProcedure<Object>() {
1281
1282                                                 @Override
1283                                                 public boolean execute(Object context, int object) {
1284                                                         result.addStatement(predicate, object);
1285                                                         return false;
1286                                                 }
1287
1288                                         }, null, clusterSupport);
1289                                 } catch (DatabaseException e) {
1290                                         e.printStackTrace();
1291                                 }
1292                                 
1293                         }
1294                 
1295         }
1296
1297         try {
1298                         cluster.forPredicates(subject, new Proc(), null, clusterSupport);
1299                 } catch (DatabaseException e) {
1300                         e.printStackTrace();
1301                 }
1302         
1303     }
1304     
1305 //      private void getDirectObjects4(final int callerThread, final ClusterI cluster, final int subject, final int predicate, final QueryProcessor processor, final ReadGraphImpl graph, final ForEachObjectProcedure procedure) {
1306 //
1307 //        if(!cluster.isLoaded()) {
1308 //
1309 //                      requestCluster(callerThread, cluster.getClusterId(), new Callback<Integer>() {
1310 //      
1311 //                              @Override
1312 //                              public void run(Integer i) {
1313 //      
1314 //                                      processor.schedule(i, new SessionTask(callerThread) {
1315 //      
1316 //                                              @Override
1317 //                                              public void run(int thread) {
1318 //                                                      
1319 //                                                      getDirectObjects4(thread, cluster, subject, predicate, processor, graph, procedure);
1320 //                                                      
1321 //                                              }
1322 //      
1323 //                                      });
1324 //      
1325 //                              }
1326 //      
1327 //                      });
1328 //                      
1329 //        } else {
1330 //
1331 //              try {
1332 //                      cluster.forObjects(graph, subject, predicate, procedure);
1333 //              } catch (DatabaseException e) {
1334 //                      e.printStackTrace();
1335 //              }
1336 //              
1337 ////                    procedure.finished(graph);
1338 ////                    graph.state.barrier.dec();
1339 ////                    
1340 ////            System.err.println("ai2=" + ai2.decrementAndGet());
1341 //              
1342 //        }
1343 //              
1344 //              
1345 //      }
1346
1347 //      AtomicInteger ai2  =new AtomicInteger(0);
1348         
1349 //    private boolean testCluster(int subject, ClusterI proxy) {
1350 //      
1351 //        if (proxy == null)
1352 //            System.out.println("null cluster: " + Integer.toString(subject, 16));
1353 //        
1354 //        return proxy != null;
1355 //      
1356 //    }
1357
1358     long getCluster(int id) {
1359         // Virtual resource
1360         if(id < 0) return 0;
1361         ClusterI proxy = clusterTable.getClusterByResourceKey(id);
1362         if(proxy == null) return 0;
1363         else return proxy.getClusterId();
1364     }
1365
1366         @Override
1367         public int getRandomAccessReference(String id) throws ResourceNotFoundException {
1368                 
1369                 try {
1370                         Resource res = serializationSupport.getResourceSerializer().getResource(id);
1371                         if(res == null) return 0;
1372                         else return ((ResourceImpl)res).id;
1373                 } catch (InvalidResourceReferenceException e) {
1374                         //e.printStackTrace();
1375                 }
1376                 return 0;
1377                 
1378         }
1379         
1380         @Override
1381         public void ensureLoaded(final ReadGraphImpl graph, final int subject, final int predicate) {
1382                 
1383                 if(subject < 0) {
1384                         
1385                         SessionImplSocket.loadVirtualStatements(virtualGraphServerSupport, graph, subject, predicate, g -> {});
1386                         
1387                 } else {
1388                         
1389                 final ClusterI cluster = clusterTable.checkedGetClusterByResourceKey(subject);
1390
1391                 if(cluster.isLoaded()) {
1392                         
1393                         if(cluster.hasVirtual() && virtualGraphServerSupport.virtuals.contains(subject) && !SessionImplSocket.areVirtualStatementsLoaded(virtualGraphServerSupport, subject, predicate)) {
1394
1395                                         SessionImplSocket.loadVirtualStatements(virtualGraphServerSupport, graph, subject, predicate, g -> {});
1396                                 
1397                                 } else {
1398                                         
1399                                 }
1400                         
1401                 } else {
1402                         
1403                                 //new Exception().printStackTrace();
1404                         
1405                         cluster.load(session.clusterTranslator, new Runnable() {
1406
1407                                         @Override
1408                                         public void run() {
1409                                                 
1410                                         if(cluster.hasVirtual() && virtualGraphServerSupport.virtuals.contains(subject) && !SessionImplSocket.areVirtualStatementsLoaded(virtualGraphServerSupport, subject, predicate)) {
1411
1412                                                         SessionImplSocket.loadVirtualStatements(virtualGraphServerSupport, graph, subject, predicate, g -> {});
1413                                                 
1414                                                 } else {
1415                                                         
1416                                                 }
1417                                                 
1418                                         }
1419                                 
1420                         });
1421                         
1422                 }
1423                 
1424                 
1425                 }
1426                 
1427         }
1428         
1429         @Override
1430         public void ensureLoaded(final ReadGraphImpl graph, final int subject) {
1431                 
1432                 if(subject < 0) {
1433                         
1434                         SessionImplSocket.loadVirtualStatements(virtualGraphServerSupport, graph, subject, g -> {});
1435                         
1436                 } else {
1437                         
1438                 final ClusterI cluster = clusterTable.getClusterByResourceKey(subject);
1439
1440                 if(cluster.isLoaded()) {
1441                         
1442                         if(cluster.hasVirtual() && virtualGraphServerSupport.virtuals.contains(subject) && !SessionImplSocket.areVirtualStatementsLoaded(virtualGraphServerSupport, subject)) {
1443
1444                                         SessionImplSocket.loadVirtualStatements(virtualGraphServerSupport, graph, subject, g -> {});
1445                                 
1446                                 } else {
1447                                         
1448                                 }
1449                         
1450                 } else {
1451                         
1452 //                      System.err.println("cluster not loaded " + subject);
1453                                 //new Exception().printStackTrace();
1454                         
1455                         cluster.load(session.clusterTranslator, new Runnable() {
1456
1457                                         @Override
1458                                         public void run() {
1459                                                 
1460                                         if(cluster.hasVirtual() && virtualGraphServerSupport.virtuals.contains(subject) && !SessionImplSocket.areVirtualStatementsLoaded(virtualGraphServerSupport, subject)) {
1461
1462                                                         SessionImplSocket.loadVirtualStatements(virtualGraphServerSupport, graph, subject, g -> {});
1463                                                 
1464                                                 } else {
1465                                                         
1466                                                 }
1467                                                 
1468                                         }
1469                                 
1470                         });
1471                         
1472                 }
1473                 
1474                 
1475                 }
1476                 
1477         }
1478
1479         @Override
1480         public boolean isLoaded(int subject) {
1481         ClusterI cluster = clusterTable.getClusterByResourceKey(subject);
1482         return cluster.isLoaded();
1483         }
1484         
1485         @Override
1486         public void ceased(int thread) {
1487                 
1488                 session.ceased(thread);
1489                 
1490         }
1491
1492     @Override
1493     public Object getLock() {
1494         
1495         return session.requestManager;
1496         
1497     }
1498
1499     @Override
1500     public VirtualGraph getProvider(int subject, int predicate, int object) {
1501
1502         if(subject > 0) {
1503             ClusterImpl cluster = (ClusterImpl)clusterTable.getClusterByResourceKey(subject);
1504                 // This persistent resource does not have virtual statements => must deny in persistent graph
1505                 if(!cluster.hasVirtual() || !virtualGraphServerSupport.virtuals.contains(subject)) return null;
1506         }
1507         
1508         for(TransientGraph g : virtualGraphServerSupport.providers) {
1509                 for (final int id : g.getObjects(subject, predicate)) {
1510                         if(object == id) return g;
1511                 }
1512         }
1513         
1514         // Nothing found from virtual graphs
1515         return null;
1516                 
1517     }
1518
1519     @Override
1520     public VirtualGraph getProvider(int subject, int predicate) {
1521
1522         if(subject > 0) {
1523             ClusterImpl cluster = (ClusterImpl)clusterTable.getClusterByResourceKey(subject);
1524                 // This persistent resource does not have virtual statements => must deny in persistent graph
1525                 if(!cluster.hasVirtual() || !virtualGraphServerSupport.virtuals.contains(subject)) return null;
1526         }
1527
1528         TransientGraph result = null;
1529         for(TransientGraph g : virtualGraphServerSupport.providers) {
1530                 if(g.getObjects(subject, predicate).length > 0) {
1531                         // Found multiple, return null;
1532                         if(result != null) return null;
1533                         else result = g;
1534                 }
1535         }
1536         return result;
1537                 
1538     }
1539
1540     @Override
1541     public VirtualGraph getValueProvider(int subject) {
1542
1543         if(subject > 0) {
1544             ClusterImpl cluster = (ClusterImpl)clusterTable.getClusterByResourceKey(subject);
1545                 // This persistent resource does not have virtual statements => must deny in persistent graph
1546                 if(!cluster.hasVirtual() || !virtualGraphServerSupport.virtuals.contains(subject)) return null;
1547         }
1548
1549         TransientGraph result = null;
1550         for(TransientGraph g : virtualGraphServerSupport.providers) {
1551                 if(g.getValue(subject) != null) {
1552                         if(result != null) return null;
1553                         else result = g;
1554                 }
1555         }
1556         return result;
1557                 
1558     }
1559     
1560     @Override
1561     public void exit(Throwable t) {
1562         state.close();
1563     }
1564     
1565     private void analyseProblem(ClusterI cluster) {
1566         
1567         if(cluster instanceof ClusterSmall)
1568                         try {
1569                                 ((ClusterSmall)cluster).check();
1570                         } catch (DatabaseException e) {
1571                                 LOGGER.error("ClusterSmall.check failed", e);
1572                         }
1573         
1574     }
1575     
1576 }