]> gerrit.simantics Code Review - simantics/interop.git/blob - org.simantics.interop/src/org/simantics/interop/test/GraphComparator.java
Process changes in smaller chunks
[simantics/interop.git] / org.simantics.interop / src / org / simantics / interop / test / GraphComparator.java
1 /*******************************************************************************
2  * Copyright (c) 2007, 2010 Association for Decentralized Information Management
3  * in Industry THTH ry.
4  * All rights reserved. This program and the accompanying materials
5  * are made available under the terms of the Eclipse Public License v1.0
6  * which accompanies this distribution, and is available at
7  * http://www.eclipse.org/legal/epl-v10.html
8  *
9  * Contributors:
10  *     Foster Wheeler Energia Oy - initial API and implementation
11  *     VTT Technical Research Centre of Finland - Improvements to comparison.
12  *     Semantum Oy - Improvements to comparison, various bug fixes. 
13  *******************************************************************************/
14 package org.simantics.interop.test;
15
16 import java.util.ArrayList;
17 import java.util.Arrays;
18 import java.util.Collection;
19 import java.util.Collections;
20 import java.util.Comparator;
21 import java.util.HashMap;
22 import java.util.HashSet;
23 import java.util.List;
24 import java.util.Map;
25 import java.util.Map.Entry;
26 import java.util.Set;
27 import java.util.Stack;
28
29 import org.eclipse.core.runtime.IProgressMonitor;
30 import org.eclipse.core.runtime.NullProgressMonitor;
31 import org.simantics.databoard.Bindings;
32 import org.simantics.db.ReadGraph;
33 import org.simantics.db.Resource;
34 import org.simantics.db.Session;
35 import org.simantics.db.Statement;
36 import org.simantics.db.common.request.ReadRequest;
37 import org.simantics.db.common.utils.NameUtils;
38 import org.simantics.db.exception.DatabaseException;
39 import org.simantics.db.request.Read;
40 import org.simantics.interop.test.GraphChanges.Modification;
41 import org.simantics.layer0.Layer0;
42 import org.simantics.utils.datastructures.BijectionMap;
43 import org.simantics.utils.datastructures.MapList;
44 import org.simantics.utils.datastructures.Pair;
45
46 /**
47  * Compares two subgraphs and reports differences.
48  * 
49  * Assumes that subgraphs (defined using traverse relations) are not cyclic.
50  * 
51  * Assumes that properties can be used to identify objects, if relation type is not enough.
52  * 
53  * 
54  * 
55  * @author Marko Luukkainen <marko.luukkainen@vtt.fi>
56  *
57  */
58 public class GraphComparator {
59         
60         private static final boolean DEBUG = false;
61
62         private Resource r1;
63         private Resource r2;
64         private Set<Resource> strong = new HashSet<>();       // List of relations that identify object, if subject is already identified. 
65         private List<Resource> traversed = new ArrayList<>(); // list of relations that are traversed (and tested)
66         private List<Resource> tested = new ArrayList<>();    // list of relations that are tested, but not traversed
67         private List<Resource> nonTraversed = new ArrayList<>(); // list of relations that are not traversed
68         private List<Resource> nonTested = new ArrayList<>(); // list of relations that are not tested
69         
70         private List<Statement> changes1 = new ArrayList<>();
71         private List<Statement> changes2 = new ArrayList<>();
72         private List<Modification> modifications = new ArrayList<>();
73         private Set<Statement> changes1Set = new HashSet<>();
74         private Set<Statement> changes2Set = new HashSet<>();
75         private Set<Modification> modificationsSet = new HashSet<>();
76
77         private BijectionMap<Statement, Statement> comparableStatements = new BijectionMap<>();
78         private BijectionMap<Resource, Resource> comparableResources = new BijectionMap<>();
79         
80         private Set<Resource> processedResources = new HashSet<Resource>();
81         
82         private ResourceComparator comparator;
83         
84         private Comparator<Statement> scomp = new PredicateComparator();
85         private Comparator<Resource> rcomp = new ResComparator();
86         
87         private Set<Resource> nonMatchedLeft = new HashSet<Resource>();
88         private Set<Resource> nonMatchedRight = new HashSet<Resource>();
89         
90         private long iter = 0;
91         private long maxIter = -1;
92         
93         // runtime attributes
94         
95         private ReadGraph g;
96         private Layer0 b;
97         
98         private boolean mapLiterals = true;
99         
100         public GraphComparator(Resource r1, Resource r2) {
101                 this.r1 = r1;
102                 this.r2 = r2;
103                 comparator = new TypeComparator();
104         }
105         
106         public GraphComparator(Resource r1, Resource r2, ResourceComparator comparator) {
107                 this.r1 = r1;
108                 this.r2 = r2;
109                 this.comparator = comparator;
110         }
111         
112         ArrayList<Statement> ss1 = new ArrayList<Statement>();
113         ArrayList<Statement> ss2 = new ArrayList<Statement>();
114         
115         
116         public Comparator<Resource> getResourceComparator() {
117                 return rcomp;
118         }
119         
120         public Comparator<Statement> getStatementComparator() {
121                 return scomp;
122         }
123         
124         public Resource getR1() {
125                 return r1;
126         }
127         
128         public Resource getR2() {
129                 return r2;
130         }
131         
132         public void addTraversed(Resource rel) {
133                 traversed.add(rel);
134         }
135         
136         public void addTraversed(Collection<Resource> rels) {
137                 traversed.addAll(rels);
138         }
139         
140         public void addNonTraversed(Resource rel) {
141                 nonTraversed.add(rel);
142         }
143         
144         public void addNonTraversed(Collection<Resource> rels) {
145                 nonTraversed.addAll(rels);
146         }
147         
148         public void addTested(Resource rel) {
149                 tested.add(rel);
150         }
151         
152         public void addTested(Collection<Resource> rels) {
153                 tested.addAll(rels);
154         }
155         
156         public void addNonTested(Resource rel) {
157                 nonTested.add(rel);
158         }
159         
160         public void addNonTested(Collection<Resource> rels) {
161                 nonTested.addAll(rels);
162         }
163         
164         public void addComparableResources(Resource r1, Resource r2) {
165                 if (DEBUG)
166                         System.out.println("Preset " + r1 + " = " + r2);
167                 comparableResources.map(r1, r2);
168         }
169         
170         public void addComparableResources(BijectionMap<Resource, Resource> matching) {
171                 if (DEBUG) {
172                         for (Entry<Resource, Resource> entry : matching.getEntries())
173                                 System.out.println("Preset " + entry.getKey() + " = " + entry.getValue());
174                 }
175                 comparableResources.addAll(matching);
176         }
177         
178         public void addStrong(Resource r) {
179                 strong.add(r);
180         }
181         
182         public void addStrong(Collection<Resource> rels) {
183                 strong.addAll(rels);
184         }
185         
186         public void addNonMatchedLeft(Resource r) {
187                 nonMatchedLeft.add(r);
188         }
189         
190         public void addNonMatchedRight(Resource r) {
191                 nonMatchedRight.add(r);
192         }
193         
194         public long getMaxIter() {
195                 return maxIter;
196         }
197         
198         /**
199          * Sets maximum iteration done on a single DB transaction. Affects only test(Session) methods.
200          * Default value is -1, which disables iteration limit.
201          * Note that using iteration limit causes comparison to repeat some work, thus total execution time will increase. 
202          * 
203          * @param maxIter
204          */
205         public void setMaxIter(long maxIter) {
206                 this.maxIter = maxIter;
207         }
208         
209         public boolean isMapLiterals() {
210                 return mapLiterals;
211         }
212         
213         public void setMapLiterals(boolean mapLiterals) {
214                 this.mapLiterals = mapLiterals;
215         }
216         
217         private void clear() {
218                 changes1.clear();
219                 changes1Set.clear();
220                 changes2.clear();
221                 changes2Set.clear();
222                 comparableResources.clear();
223                 comparableStatements.clear();
224                 modifications.clear();
225                 modificationsSet.clear();
226                 processedResources.clear();
227         }
228         
229         public void dispose() {
230                 changes1 = null;
231                 changes1Set = null;
232                 changes2 = null;
233                 changes2Set = null;
234                 comparableResources = null;
235                 comparableStatements = null;
236                 comparator = null;
237                 modifications = null;
238                 modificationsSet = null;
239                 processedResources = null;
240                 nonMatchedLeft = null;
241                 nonMatchedRight = null;
242                 nonTested = null;
243                 nonTraversed = null;
244                 ss1 = null;
245                 ss2 = null;
246                 strong = null;
247                 tested = null;
248                 traversed = null;
249         }
250         
251         public void test(ReadGraph g) throws DatabaseException {
252                 test(g,null);
253         }
254         
255         public void test(ReadGraph g, IProgressMonitor monitor) throws DatabaseException {
256                 this.g = g;
257                 this.b = Layer0.getInstance(g);
258                 comparator.setComparator(this);
259                 comparator.initialize(g, r1, r2);
260                 
261                 if (monitor == null)
262                         monitor = new NullProgressMonitor();
263                 
264                 Stack<Resource> objectsLeft = new Stack<Resource>();
265                 Stack<Resource> objectsRight = new Stack<Resource>();
266                 objectsLeft.push(r1);
267                 objectsRight.push(r2);
268                 
269                 Set<Statement> unreliableLeft = new HashSet<Statement>();
270                 Set<Statement> unreliableRight = new HashSet<Statement>();
271                 
272                 boolean changed = false;
273                 while (true) {
274                         if (objectsLeft.isEmpty() && !changed)
275                                 break;
276                         if (monitor.isCanceled()) {
277                                 clear();
278                                 return;
279                         }
280                         monitor.subTask(monitorTaskName(objectsLeft, objectsRight, unreliableLeft, unreliableRight));
281                         changed = false;
282                         
283                         // process compares objects that are identified and searches for more resources to process. 
284                         changed = compareIter(g, objectsLeft, objectsRight, unreliableLeft, unreliableRight);
285                         monitor.worked(1);
286                         
287                 }
288                 for (Statement s : unreliableLeft) {
289                         if (!comparableStatements.containsLeft(s)) {
290                                 if (DEBUG) System.out.println("Unreliable Object deletion " + printStatement(g,s));
291                                 addDeletion(s);
292                         }
293                 }
294                 for (Statement s : unreliableRight) {
295                         if (!comparableStatements.containsRight(s)) {
296                                 if (DEBUG) System.out.println("Unreliable Object addition " + printStatement(g,s));
297                                 addAddition(s);
298                         }
299                 }
300                 monitor.subTask(monitorTaskName(objectsLeft, objectsRight, unreliableLeft, unreliableRight) + " done.");
301                 //printStats();
302         }
303         
304         public void test(Session session) throws DatabaseException {
305                 test(session, r1, r2, null);
306         }
307         
308         public void test(Session session, IProgressMonitor monitor) throws DatabaseException {
309                 test(session, r1, r2, monitor);
310         }
311         
312         public void test(Session session, Resource r1, Resource r2) throws DatabaseException {
313                 test(session, r1, r2, null);
314         }
315         
316         public void test(Session session, Resource r1, Resource r2, IProgressMonitor monitor) throws DatabaseException {
317                 
318                 comparator.setComparator(this);
319                 
320                 if (monitor == null)
321                         monitor = new NullProgressMonitor();
322                 
323                 session.syncRequest(new ReadRequest() {
324             
325             @Override
326             public void run(ReadGraph graph) throws DatabaseException {
327                 comparator.initialize(graph, r1, r2);
328             }
329         });
330                 
331                 addComparable(r1, r2);
332                 
333                 final Stack<Resource> objectsLeft = new Stack<Resource>();
334                 final Stack<Resource> objectsRight = new Stack<Resource>();
335                 objectsLeft.push(r1);
336                 objectsRight.push(r2);
337                 
338                 final Set<Statement> unreliableLeft = new HashSet<Statement>();
339                 final Set<Statement> unreliableRight = new HashSet<Statement>();
340                 
341                 boolean changed = false;
342                 while (true) {
343                         if (objectsLeft.isEmpty() && !changed)
344                                 break;
345                         if (monitor.isCanceled()) {
346                                 clear();
347                                 return;
348                         }
349                         monitor.subTask(monitorTaskName(objectsLeft, objectsRight, unreliableLeft, unreliableRight));
350                         changed = session.syncRequest(new Read<Boolean>() {
351                                 
352                                 @Override
353                                 public Boolean perform(ReadGraph graph) throws DatabaseException {
354                                         //System.out.println("Iter " + monitorTaskName(objectsLeft, objectsRight, unreliableLeft, unreliableRight));
355                                         g = graph;
356                                         b = Layer0.getInstance(graph);
357                                         return compareIter(graph, objectsLeft, objectsRight, unreliableLeft, unreliableRight);
358                                 }
359                         });
360                         monitor.worked(1);
361                         
362                         
363                 }
364                 for (Statement s : unreliableLeft) {
365                         if (!comparableStatements.containsLeft(s))
366                                 addDeletion(s);
367                 }
368                 for (Statement s : unreliableRight) {
369                         if (!comparableStatements.containsRight(s))
370                                 addAddition(s);
371                 }
372                 unreliableLeft.clear();
373                 unreliableRight.clear();
374                 
375                 monitor.subTask(monitorTaskName(objectsLeft, objectsRight, unreliableLeft, unreliableRight) + " done.");
376                 //printStats();
377         }
378         
379         private boolean compareIter(ReadGraph graph, Stack<Resource> objectsLeft, Stack<Resource> objectsRight, Set<Statement> unreliableLeft, Set<Statement> unreliableRight) throws DatabaseException {
380                 // process compares objects that are identified and searches for more resources to process.
381                 iter = 0;
382                 boolean c = process(objectsLeft, objectsRight, unreliableLeft, unreliableRight);
383                 if (objectsLeft.isEmpty()) {
384                         // process unreliable handles cases where unidentified statements subject and object have been identified 
385                         c |= processUnreliable(unreliableLeft, unreliableRight);
386                         // process unreliable handles cases where unidentified resources have path of length one to identified resource
387                         if (!c) {
388                                 c |= processUnreliable(unreliableLeft, unreliableRight,objectsLeft,objectsRight);
389                                 if (!c && objectsLeft.isEmpty() && unreliableLeft.size() > 0 && unreliableRight.size() > 0) {
390                                         c |= processUnreliable2(unreliableLeft, unreliableRight,objectsLeft,objectsRight);
391                                 }
392                                 if (!c && objectsLeft.isEmpty() && unreliableLeft.size() > 0 && unreliableRight.size() > 0) {
393                                         // comparison is ending, but we have still unprocessed unidentified resources left.
394                                         // These cases have longer path than one to identified objects.
395                                         c |= processUnreliableDeep(unreliableLeft, unreliableRight, objectsLeft, objectsRight);
396                                 }
397                                 if (!c && objectsLeft.isEmpty() && unreliableLeft.size() > 0 && unreliableRight.size() > 0) {
398                                         // comparison is ending, but we have still unprocessed unidentified resources left.
399                                         // These cases have longer path than one to identified objects.
400                                         c |= processUnreliableDeep(unreliableLeft, unreliableRight, objectsLeft, objectsRight);
401                                 }
402                         }
403                 }
404                 return c;               
405                 
406         }
407         
408         private String monitorTaskName(Stack<Resource> objectsLeft, Stack<Resource> objectsRight, Set<Statement> unreliableLeft, Set<Statement> unreliableRight) {
409                 int cr = comparableResources.size();
410                 int ch = Math.max(changes1.size(), changes2.size());
411                 return "Graph compare " + (cr + ch) +  " / " + (cr+ch+(Math.max(objectsLeft.size(), objectsRight.size())+Math.max(unreliableLeft.size(), unreliableRight.size())));
412         }
413         
414         private boolean process(Stack<Resource> objectsLeft, Stack<Resource> objectsRight, Set<Statement> unreliableLeft, Set<Statement> unreliableRight) throws DatabaseException {
415                 List<Statement> ss1 = new ArrayList<Statement>();
416                 List<Statement> ss2 = new ArrayList<Statement>();
417                 boolean didSomething = false;
418                 while (!objectsLeft.isEmpty()&& (maxIter < 0 || iter < maxIter)) {
419                         Resource r1 = objectsLeft.pop();
420                         Resource r2 = objectsRight.pop();
421                         
422                         if (r1.equals(r2))
423                                 continue;
424                         if (processedResources.contains(r1))
425                                 continue;
426                         processedResources.add(r1);
427                         
428                 
429                         if((comparableResources.containsLeft(r1)||comparableResources.containsRight(r2)) && !comparableResources.contains(r1, r2)) {
430                                 throw new DatabaseException("Comparator error: Trying to map " + r1 + " to " + r2 + " while mappings " + r1 + " to " + comparableResources.getRight(r1) + " and " + comparableResources.getLeft(r2) + " to " + r2 + " exist.");
431                         }
432                         addComparable(r1, r2);
433                         didSomething = true;
434                         //System.out.println("test " + NameUtils.getSafeName(g, r1) + " " + NameUtils.getSafeName(g, r2));
435                         compareProps(r1, r2);
436                         
437                         for (Resource rel : tested) {
438                                 ss1.addAll(g.getStatements(r1, rel));
439                                 ss2.addAll(g.getStatements(r2, rel));
440                                 ss1 = filterAsserted(r1, ss1);
441                                 ss2 = filterAsserted(r2, ss2);
442                                 ss1 = filterTraversed(ss1);
443                                 ss2 = filterTraversed(ss2);
444                                 ss1 = filterNonTested(ss1);
445                                 ss2 = filterNonTested(ss2);
446                                 ss1 = filter(Collections.singletonList(b.HasProperty), ss1);
447                                 ss2 = filter(Collections.singletonList(b.HasProperty), ss2);
448                                 
449                                 compareStatements(ss1, ss2, null, null,null,null);
450                                 ss1.clear();
451                                 ss2.clear();
452                         }
453                         
454                         for (Resource rel : traversed) {
455                                 ss1.addAll(g.getStatements(r1, rel));
456                                 ss2.addAll(g.getStatements(r2, rel));
457                                 ss1 = filterAsserted(r1, ss1);
458                                 ss2 = filterAsserted(r2, ss2);
459                                 ss1 = filterNonTraversed(ss1);
460                                 ss2 = filterNonTraversed(ss2);
461                                 compareStatements(ss1, ss2, objectsLeft, objectsRight,unreliableLeft,unreliableRight);
462                                 ss1.clear();
463                                 ss2.clear();
464                                 
465                         }
466                         iter++;
467                 }
468                 return didSomething;
469         }
470         
471         private boolean processUnreliable(Set<Statement> unreliableLeft, Set<Statement> unreliableRight) throws DatabaseException {
472                 MapList<Resource,Statement> subjectLeft = new MapList<Resource, Statement>();
473                 MapList<Resource,Statement> subjectRight = new MapList<Resource, Statement>();
474                 MapList<Resource,Statement> objectLeft = new MapList<Resource, Statement>();
475                 MapList<Resource,Statement> objectRight = new MapList<Resource, Statement>();
476                 boolean didSomething = false;
477                 for (Statement s : unreliableLeft) {
478                         subjectLeft.add(s.getSubject(),s);
479                         objectLeft.add(s.getObject(),s);
480                 }
481                 for (Statement s : unreliableRight) {
482                         subjectRight.add(s.getSubject(),s);
483                         objectRight.add(s.getObject(),s);
484                 }
485                 
486                 for (Resource left : subjectLeft.getKeys()) {
487                         if (maxIter > 0 && iter > maxIter)
488                                 break;
489                         Resource right = comparableResources.getRight(left);
490                         if (right == null)
491                                 continue;
492                         for (Statement leftS : subjectLeft.getValues(left)) {
493                                 Resource leftO = leftS.getObject();
494                                 if (!unreliableLeft.contains(leftS))
495                                         continue;
496                                 Resource rightO = comparableResources.getRight(leftO);
497                                 if (rightO == null)
498                                         continue;
499                                 for (Statement rightS : subjectRight.getValues(right)) {
500                                         if (!rightS.getObject().equals(rightO))
501                                                 continue;
502                                         if (!unreliableRight.contains(rightS))
503                                                 continue;
504                                         if (leftS.getPredicate().equals(rightS.getPredicate()) ||
505                                                 comparableResources.contains(leftS.getPredicate(), rightS.getPredicate())) {
506                                                 unreliableLeft.remove(leftS);
507                                                 unreliableRight.remove(rightS);
508                                                 addComparable(leftS, rightS);
509                                                 didSomething = true;
510                                                 iter++;
511                                         }
512                                 }
513                         }               
514                 }
515                 return didSomething;
516         }
517         
518         private boolean processUnreliable(Set<Statement> unreliableLeft, Set<Statement> unreliableRight, Stack<Resource> objectsLeft, Stack<Resource> objectsRight) throws DatabaseException {
519                 MapList<Resource,Statement> subjectLeft = new MapList<Resource, Statement>();
520                 MapList<Resource,Statement> subjectRight = new MapList<Resource, Statement>();
521                 MapList<Resource,Statement> objectLeft = new MapList<Resource, Statement>();
522                 MapList<Resource,Statement> objectRight = new MapList<Resource, Statement>();
523                 boolean didSomething = false;
524                 for (Statement s : unreliableLeft) {
525                         subjectLeft.add(s.getSubject(),s);
526                         objectLeft.add(s.getObject(),s);
527                 }
528                 for (Statement s : unreliableRight) {
529                         subjectRight.add(s.getSubject(),s);
530                         objectRight.add(s.getObject(),s);
531                 }
532                 for (Resource ol : objectLeft.getKeys()) {
533                         if (maxIter > 0 && iter > maxIter)
534                                 break;
535                         // all statements to the left side object
536                         List<Statement> left = objectLeft.getValues(ol);
537                         // all subjects that have statements to the left side object (ol)
538                         Set<Resource> sLeft = new HashSet<Resource>();
539                         // all matching subjects on the right side
540                         Set<Resource> sRight = new HashSet<Resource>();
541                         for (Statement s : left) {
542                                 sLeft.add(s.getSubject());
543                                 sRight.add(comparableResources.getRight(s.getSubject()));
544                         }
545                         
546                         // check if object left can be reliably identified by available statements
547                         // if there are any objects on the left side with similar statements, object left cannot be mapped.
548                         boolean hasSimilar = false;
549                         MapList<Resource, Statement> comparableOLeft = new MapList<Resource, Statement>();
550                         for (Resource sl : sLeft) {
551                                 for (Statement s : subjectLeft.getValues(sl)) {
552                                         if (!s.getObject().equals(ol)) {
553                                                 comparableOLeft.add(s.getObject(),s);
554                                         }
555                                 }
556                         }
557                         
558                         compareStatements(ss1, ss2, objectsLeft, objectsRight,unreliableLeft,unreliableRight);
559                         
560                         for (Resource similarOl : comparableOLeft.getKeys()) {
561                                 List<Statement> similarLeft = comparableOLeft.getValues(similarOl);
562                                 if (similarLeft.size() == left.size()) {
563                                         boolean useL[] = new boolean[left.size()];
564                                         boolean useSL[] = new boolean[left.size()];
565                                         for (int i = 0; i < left.size(); i++) {
566                                                 useL[i] = false;
567                                                 useSL[i] = false;
568                                         }
569                                         for (int i = 0; i < left.size(); i++) {
570                                                 for (int j = 0; j < left.size(); j++) {
571                                                         if (useSL[j])
572                                                                 continue;
573                                                         // compare predicates
574                                                         Resource pl = left.get(i).getPredicate();
575                                                         Resource psl = similarLeft.get(j).getPredicate();
576                                                         if (pl.equals(psl)) {
577                                                                 // compare objects (unreliable result is interpreted as positive match)
578
579                                                                 int comp = comparator.compare(g, left.get(i).getObject(), similarLeft.get(j).getObject(), true);
580                                                                 if (comp >= 0 && comp < ResourceComparator.NO_MATCH) {
581                                                                         useL[i] = true;
582                                                                         useSL[j] = true;
583                                                                         break;
584                                                                 }
585                                                         }
586                                                 }
587                                         }
588                                         boolean diff = false;
589                                         for (int i = 0; i < left.size(); i++) {
590                                                 if (!useL[i] || !useSL[i]) {
591                                                         diff = true;
592                                                 }
593                                         }
594                                         if (!diff) {
595                                                 hasSimilar = true;
596                                                 break;
597                                         }
598                                 }
599                         }
600                         
601                         if (hasSimilar)
602                                 continue;
603                                 
604                         
605                         // all objects that subjects on the right side point to. Object left has its matching resource among these, if it has matching resource
606                         MapList<Resource,Statement> possibleOR = new MapList<Resource, Statement>();
607                         for (Resource sr : sRight) {
608                                 for (Statement s : subjectRight.getValues(sr))
609                                         possibleOR.add(s.getObject(),s);
610                         }
611                         
612                         // filter possible right side objects to those that have same amount of statements as the left side object
613                         for (Resource or : possibleOR.getKeys().toArray(new Resource[possibleOR.getKeys().size()])) {
614                                 List<Statement> right = possibleOR.getValues(or);
615                                 if (right.size() != left.size())
616                                         possibleOR.remove(or);
617                                         
618                         }
619                         
620                         // check for matching statements (comparable subjects, matching predicates)
621                         MapList<Resource,Statement> matchingOR = new MapList<Resource, Statement>(); // list of objects that have matching statements
622                         Map<Resource,Pair<int[], int[]>> matchingStatements = new HashMap<Resource, Pair<int[], int[]>>(); // matching statements
623                         for (Resource or : possibleOR.getKeys()) {
624                                 List<Statement> right = possibleOR.getValues(or);
625                                 int iLeft[] = new int[left.size()];
626                                 int iRight[] = new int[right.size()];
627                                 
628                                 for (int i = 0; i < left.size(); i++) {
629                                         iLeft[i] = -1;
630                                         iRight[i] = -1;
631                                 }
632                                 
633                                 for (int l = 0; l < left.size(); l++) {
634                                         Statement ls = left.get(l);
635                                         for (int r = 0; r < right.size(); r++) {
636                                                 if (iRight[r] >= 0)
637                                                         continue;
638                                                 Statement rs = right.get(r);
639                                                 if (!comparableResources.contains(ls.getSubject(), rs.getSubject()))
640                                                         continue;
641                                                 if ((comparableResources.containsLeft(ls.getObject()) || comparableResources.containsRight(rs.getObject())) && !comparableResources.contains(ls.getObject(), rs.getObject()))
642                                                         continue;
643                                                 if (rcomp.compare(ls.getPredicate(),rs.getPredicate()) == 0) {
644                                                         // compare objects (unreliable result is not accepted)
645                                                         int comp = comparator.compare(g, ls.getObject(), rs.getObject());
646                                                         if (comp > 0 && comp < Integer.MAX_VALUE) {
647                                                                 iLeft[l] = r;
648                                                                 iRight[r] = l;
649                                                                 break;
650                                                         }
651                                                         break;
652                                                 }
653                                         }
654                                         
655                                 }
656                                 boolean success = true;
657                                 for (int i = 0; i < left.size(); i++) {
658                                         if (iLeft[i] < 0) {
659                                                 success = false;
660                                                 break;
661                                         }
662                                         if (iRight[i] < 0) {
663                                                 success = false;
664                                                 break;
665                                         }
666                                                 
667                                 }
668                                 if (success) {
669                                         for (Statement s : right) 
670                                                 matchingOR.add(or,s);
671                                         matchingStatements.put(or, new Pair<int[], int[]>(iLeft, iRight));
672                                 }
673                         }
674                         // if there is only one matching right side object, we have found a match 
675                         if (matchingOR.getKeySize() == 1) {
676                                 Resource or = matchingOR.getKeys().iterator().next();
677                                 List<Statement> right = matchingOR.getValues(or);
678                                 Pair<int[], int[]> indices = matchingStatements.get(or);
679                                 
680                                 objectsLeft.add(ol);
681                                 objectsRight.add(or);
682                                 addComparable(ol, or);
683                                 didSomething = true;
684                                 for (int l = 0; l < left.size(); l++) {
685                                         int r = indices.first[l];
686                                         Statement sl = left.get(l);
687                                         Statement sr = right.get(r);
688                                         addComparable(sl, sr);
689                                         unreliableLeft.remove(sl);
690                                         unreliableRight.remove(sr);
691                                 }
692                                 iter++;
693                                 
694                         } else {
695                                 
696                         }
697
698                 }
699                 return didSomething;
700                 
701                 
702         }
703         
704         private boolean processUnreliable2(Set<Statement> unreliableLeft, Set<Statement> unreliableRight, Stack<Resource> objectsLeft, Stack<Resource> objectsRight) throws DatabaseException {
705                 MapList<Resource,Statement> subjectLeft = new MapList<Resource, Statement>();
706                 MapList<Resource,Statement> subjectRight = new MapList<Resource, Statement>();
707                 MapList<Resource,Statement> objectLeft = new MapList<Resource, Statement>();
708                 MapList<Resource,Statement> objectRight = new MapList<Resource, Statement>();
709                 boolean didSomething = false;
710                 for (Statement s : unreliableLeft) {
711                         subjectLeft.add(s.getSubject(),s);
712                         objectLeft.add(s.getObject(),s);
713                 }
714                 for (Statement s : unreliableRight) {
715                         subjectRight.add(s.getSubject(),s);
716                         objectRight.add(s.getObject(),s);
717                 }
718                 Set<Pair<Resource, Resource>> processed = new HashSet<Pair<Resource,Resource>>();
719                 for (Resource ol : objectLeft.getKeys()) {
720                         if (maxIter > 0 && iter > maxIter)
721                                 break;
722                         // all statements to the left side object
723                         List<Statement> left = objectLeft.getValues(ol);
724                         // all subjects that have statements to the left side object (ol)
725                         Set<Resource> sLeft = new HashSet<Resource>();
726                         // all matching subjects on the right side
727                         Set<Resource> sRight = new HashSet<Resource>();
728                         for (Statement s : left) {
729                                 sLeft.add(s.getSubject());
730                                 sRight.add(comparableResources.getRight(s.getSubject()));
731                         }
732                         
733                         if (sLeft.size() == 1 && sRight.size() == 1) {
734                                 Resource sl = sLeft.iterator().next();
735                                 Resource sr = sRight.iterator().next();
736                                 Pair<Resource, Resource> p = new Pair<Resource, Resource>(sl, sr);
737                                 if (processed.contains(p))
738                                         continue;
739                                 processed.add(p);
740                                 List<Statement> ss1 = new ArrayList<Statement>(subjectLeft.getValuesSnapshot(sl));
741                                 List<Statement> ss2 = new ArrayList<Statement>(subjectRight.getValuesSnapshot(sr));
742                                 for (int i = ss1.size() -1; i >= 0; i--) {
743                                         if (!unreliableLeft.contains(ss1.get(i)))
744                                                 ss1.remove(i);
745                                 }
746                                 for (int i = ss2.size() -1; i >= 0; i--) {
747                                         if (!unreliableRight.contains(ss2.get(i)))
748                                                 ss2.remove(i);
749                                 }
750                                 
751                                 int ccount = comparableStatements.size();
752                                 int lcount = changes1.size();
753                                 int rcount = changes2.size();
754                                 
755                                 compareStatements(ss1, ss2, objectsLeft, objectsRight,unreliableLeft,unreliableRight);
756                                 if (comparableStatements.size() > ccount) {
757                                         didSomething = true;
758                                         for (Entry<Statement, Statement> entry : comparableStatements.getEntries()) {
759                                                 unreliableLeft.remove(entry.getKey());
760                                                 unreliableRight.remove(entry.getValue());
761                                                 iter++;
762                                         }
763                                 }
764                                 if (changes1.size() > lcount) {
765                                         didSomething = true;
766                                         for (Statement stm : changes1) {
767                                                 unreliableLeft.remove(stm);
768                                                 iter++;
769                                         }
770                                 }
771                                 if (changes2.size() > rcount) {
772                                         didSomething = true;
773                                         for (Statement stm : changes2) {
774                                                 unreliableRight.remove(stm);
775                                                 iter++;
776                                         }
777                                 }
778                         }
779                 }
780                 return didSomething;
781         }
782         
783         private boolean processUnreliableDeep(Set<Statement> unreliableLeft, Set<Statement> unreliableRight, Stack<Resource> objectsLeft, Stack<Resource> objectsRight) throws DatabaseException {
784                 MapList<Resource,Statement> subjectLeft = new MapList<Resource, Statement>();
785                 MapList<Resource,Statement> subjectRight = new MapList<Resource, Statement>();
786                 MapList<Resource,Statement> objectLeft = new MapList<Resource, Statement>();
787                 MapList<Resource,Statement> objectRight = new MapList<Resource, Statement>();
788                 boolean didSomething = false;
789                 for (Statement s : unreliableLeft) {
790                         subjectLeft.add(s.getSubject(),s);
791                         objectLeft.add(s.getObject(),s);
792                 }
793                 for (Statement s : unreliableRight) {
794                         subjectRight.add(s.getSubject(),s);
795                         objectRight.add(s.getObject(),s);
796                 }
797                 for (Resource ol : objectLeft.getKeys()) {
798                         if (maxIter > 0 && iter > maxIter)
799                                 break;
800                         Set<Path> pathsLeft = new HashSet<Path>();
801                         for (Resource rel : traversed) {
802                                 pathsLeft.addAll(Path.create(g.getStatements(ol, rel)));
803                         }
804                         while (true) {
805                                 expand(pathsLeft);
806                                 if (pathsLeft.size() == 0)
807                                         break;
808                                 Collection<Path> endPaths = new ArrayList<Path>(1);
809                                 for (Path p : pathsLeft) {
810                                         if (comparableResources.containsLeft(p.getEnd())) {
811                                                 endPaths.add(p);
812                                         }
813                                 }
814                                 if (endPaths.size() > 0) {
815                                         pathsLeft.clear();
816                                         pathsLeft.addAll(endPaths);
817                                         break;
818                                 }       
819                         }
820                         if (pathsLeft.size() > 0) {
821                                 Resource sl = objectLeft.getValues(ol).get(0).getSubject();
822                                 Resource sr = comparableResources.getRight(sl);
823                                 Collection<Resource> possibleOR = new ArrayList<Resource>();
824                                 for (Statement s : subjectRight.getValues(sr)) {
825                                         possibleOR.add(s.getObject());
826                                 }
827                                 Map<Resource,Set<Path>> matchingPaths = new HashMap<Resource, Set<Path>>();
828                                 for (Resource or : possibleOR) {
829                                         Set<Path> possiblePathsRight = new HashSet<Path>();
830                                         for (Path leftPath : pathsLeft) {
831                                                 possiblePathsRight.addAll(findComparableRight(leftPath, or));
832                                         }
833                                         if (hasMatchingPaths(pathsLeft, possiblePathsRight)) {
834                                                 matchingPaths.put(or, possiblePathsRight);
835                                         }
836                                 }
837                                 if (matchingPaths.size() > 0) {
838                                         if (matchingPaths.size() == 1) {
839                                                 didSomething = true;
840                                                 Resource or = matchingPaths.keySet().iterator().next();
841                                                 
842                                                 objectsLeft.add(ol);
843                                                 objectsRight.add(or);
844                                                 addComparable(ol, or);
845                                                 Collection<Statement> statementsLeft = objectLeft.getValues(ol);
846                                                 Collection<Statement> statementsRight = objectRight.getValues(or);
847                                                 unreliableLeft.removeAll(statementsLeft);
848                                                 unreliableRight.removeAll(statementsRight);
849                                                 BijectionMap<Path,Path> map = getMatchingPaths(pathsLeft, matchingPaths.get(or));
850                                                 for (Path left : map.getLeftSet()) {
851                                                         Path right = map.getRight(left);
852                                                         for (int i = 0; i < left.getLength(); i++) {
853                                                                 addComparable(left.getStatements().get(i),right.getStatements().get(i));
854                                                                 iter++;
855                                                         }
856                                                 }
857                                                 iter++;
858                                         } 
859                                 }
860                         }
861                         
862                 }
863                 return didSomething;
864                 
865         }
866         
867         private boolean hasMatchingPaths(Set<Path> leftPaths, Set<Path> rightPaths) throws DatabaseException {
868                 if (leftPaths.size() != rightPaths.size())
869                         return false;
870                 BijectionMap<Path,Path> map = getMatchingPaths(leftPaths, rightPaths);
871                 return map.size() == leftPaths.size();
872         }
873         
874         private BijectionMap<Path,Path> getMatchingPaths(Set<Path> leftPaths, Set<Path> rightPaths) throws DatabaseException {
875                 BijectionMap<Path,Path> map = new BijectionMap<Path, Path>();
876                 for (Path leftPath : leftPaths) {
877                         for (Path rightPath : rightPaths) {
878                                 if (map.containsRight(rightPath))
879                                         continue;
880                                 if (leftPath.getLength() != rightPath.getLength())
881                                         continue;
882                                 if (comparableResources.contains(leftPath.getEnd(), rightPath.getEnd())) {
883                                         boolean match = true;
884                                         for (int i = 0; i < leftPath.getLength(); i++) {
885                                                 Statement sl = leftPath.getStatements().get(i);
886                                                 Statement sr = rightPath.getStatements().get(i);
887                                                 if (!sl.getPredicate().equals(sr.getPredicate()) && !comparableResources.contains(sl.getPredicate(), sr.getPredicate())) {
888                                                         match = false;
889                                                         break;
890                                                 }
891                                                 if ((getComparableResources().containsLeft(sl.getObject()) || getComparableResources().containsRight(sr.getObject())) && !getComparableResources().contains(sl.getObject(), sr.getObject())) {
892                                                         match = false;
893                                                         break;
894                                                 }
895                                                 if (comparator.compare(g, sl.getObject(), sr.getObject()) == ResourceComparator.NO_MATCH) {
896                                                         match = false;
897                                                         break;
898                                                 }
899                                         }
900                                         if (match) {
901                                                 map.map(leftPath, rightPath);
902                                                 break;
903                                         }
904                                 }
905                         }
906                 }
907                 return map;
908         }
909         
910         private void expand(Set<Path> paths) throws DatabaseException {
911                 Set<Path> stepPathsLeft = new HashSet<Path>();
912                 if (paths.size() == 0)
913                         return;
914                 int length = paths.iterator().next().getLength() + 1;
915                 for (Path p : paths) {
916                         for (Resource rel : traversed) {
917                                 stepPathsLeft.addAll(Path.expand(p,g.getStatements(p.getEnd(), rel)));
918                         }
919                 }
920                 paths.clear();
921                 for (Path p : stepPathsLeft) {
922                         if (p.getLength() == length)
923                                 paths.add(p);
924                 }
925         }
926         
927         private Collection<Path> findComparableRight(Path leftPath, Resource beginRight) throws DatabaseException {
928                 Set<Path> rightPaths = new HashSet<Path>();
929                 rightPaths.addAll(Path.create(g.getStatements(beginRight, getRight(leftPath.getStatements().get(0).getPredicate()))));
930                 for (int i = 1; i < leftPath.getLength(); i++) {
931                         if (rightPaths.size() == 0)
932                                 return rightPaths;
933                         Set<Path> stepPaths = new HashSet<Path>();
934                         for (Path p : rightPaths) {
935                                 stepPaths.addAll(Path.expand(p, g.getStatements(p.getEnd(), getRight(leftPath.getStatements().get(i).getPredicate()))));
936                         }
937                         rightPaths.clear();
938                         for (Path p : stepPaths)
939                                 if (p.getLength() == i+1) 
940                                         rightPaths.add(p);
941                 }
942                 return rightPaths;
943                 
944         }
945         
946         private Resource getRight(Resource r) {
947                 if (comparableResources.containsLeft(r))
948                         return comparableResources.getRight(r);
949                 return r;
950         }
951         
952
953         
954         public BijectionMap<Statement, Statement> getComparableStatements() {
955                 return comparableStatements;
956         }
957         
958         public BijectionMap<Resource, Resource> getComparableResources() {
959                 return comparableResources;
960         }
961         
962         public GraphChanges getChanges() {
963                 return new GraphChanges(r1,r2,changes1,changes2,modifications,comparableResources);
964         }
965         
966         public List<Statement> getChanges1() {
967                 return changes1;
968         }
969         
970         public List<Statement> getChanges2() {
971                 return changes2;
972         }
973         
974         private void addComparable(Statement left, Statement right) throws DatabaseException {
975                 addComparable(left.getObject(), right.getObject());
976                 comparableStatements.map(left, right);
977         }
978         
979         private void addComparable(Resource left, Resource right) throws DatabaseException {
980                 if(!comparableResources.contains(left, right)) {
981                         if (comparableResources.containsLeft(left)||comparableResources.containsRight(right)) {
982                                 throw new DatabaseException("Comparator error: Trying to map " + left + " to " + right + " while mappings " + left + " to " + comparableResources.getRight(left) + " and " + comparableResources.getLeft(right) + " to " + right + " exist.");
983                         } else {
984                                 if (DEBUG) System.out.println(left + " = " + right);
985                                 comparableResources.map(left, right);
986                                 
987 //                              if (comparableResources.size() % 1000 == 0)
988 //                                      printStats();
989                         }
990                 }
991                 
992         }
993         
994         public void printStats() {
995                 System.out.println("Comp " + comparableResources.size() + " L " + changes1.size() + " R " + changes2.size() + " M " + modifications.size() + " P " + processedResources.size());
996         }
997         
998         public List<Statement> filterAsserted(Resource r, Collection<Statement> in) throws DatabaseException {
999                 List<Statement> out = new ArrayList<Statement>();
1000                 for (Statement s : in) {
1001                         if (!s.isAsserted(r))
1002                                 out.add(s);
1003                         
1004                 }
1005                 return out;
1006         }
1007         
1008         public List<Statement> filterAssertedDuplicates(Resource r, List<Statement> in) throws DatabaseException {
1009                 List<Statement> out = new ArrayList<Statement>();
1010                 for (int i = 0; i < in.size(); i++) {
1011                         Statement s = in.get(i);
1012                         if (!s.isAsserted(r))
1013                                 out.add(s);
1014                         else {
1015                                 boolean has = false;
1016                                 if (i > 0 && in.get(i-1).getPredicate().equals(s.getPredicate()))
1017                                         has = true;
1018                                 else if (i < in.size()-1 && in.get(i+1).getPredicate().equals(s.getPredicate()))
1019                                         has = true;
1020                                 if (!has)
1021                                         out.add(s);
1022                         }
1023                         
1024                 }
1025                 return out;
1026         }
1027         
1028         
1029
1030         private String printStatement(ReadGraph graph, Statement s) throws DatabaseException {
1031                 return NameUtils.getSafeName(graph, s.getSubject()) + " " + NameUtils.getSafeName(graph, s.getPredicate()) + " " + NameUtils.getSafeName(graph, s.getObject());
1032         }
1033         
1034         private List<Statement> filterTraversed(List<Statement> in) throws DatabaseException {
1035                 return filter(traversed, in);
1036         }
1037         
1038         private List<Statement> filterNonTested(List<Statement> in) throws DatabaseException {
1039                 return filter(nonTested, in);
1040         }
1041         
1042         private List<Statement> filterNonTraversed(List<Statement> in) throws DatabaseException {
1043                 return filter(nonTraversed, in);
1044         }
1045         
1046         private List<Statement> filter(Collection<Resource> toFilter, List<Statement> in) throws DatabaseException {
1047                 if (toFilter.size() == 0)
1048                         return in;
1049                 List<Statement> out = new ArrayList<Statement>();
1050                 for (Statement s : in) {
1051                         boolean usable = true;
1052                         for (Resource r : toFilter) {
1053                                 if (g.isSubrelationOf(s.getPredicate(),r)) {
1054                                         usable = false;
1055                                         break;
1056                                 }
1057                         }
1058                         if (usable) {
1059                                 out.add(s);
1060                         }
1061                         
1062                 }
1063                 return out;
1064         }
1065         
1066         
1067         private void addDeletion(Statement s) {
1068                 if (!changes1Set.contains(s)) {
1069                         if (DEBUG) System.out.println("- " +s.getSubject() + " " + s.getPredicate() + " " + s.getObject()) ;
1070                         changes1Set.add(s);
1071                         changes1.add(s);
1072                 }
1073         }
1074         
1075         private void addAddition(Statement s) {
1076                 if (!changes2Set.contains(s)) {
1077                         changes2Set.add(s);
1078                         if (DEBUG) System.out.println("+ " +s.getSubject() + " " + s.getPredicate() + " " + s.getObject()) ;
1079                         changes2.add(s);
1080                 }
1081         }
1082         
1083         private void addModification(Resource left, Statement leftstm, Resource right, Statement rightstm) {
1084                 Modification mod = new Modification(left, right, leftstm, rightstm);
1085                 if (!modificationsSet.contains(mod)) {
1086                         modificationsSet.add(mod);
1087                         modifications.add(mod);
1088                 }
1089         }
1090         
1091         public void sortStatement(List<Statement> list1, List<Statement> list2) {
1092                 sortStatement(list1, list2, scomp);
1093         }
1094         
1095         public void sortStatement(List<Statement> list1, List<Statement> list2, Comparator<Statement> scomp) {
1096                 Collections.sort(list1,scomp);
1097                 Collections.sort(list2,scomp);
1098                 
1099                 List<Statement> sorted1 = new ArrayList<Statement>(list1.size());
1100                 List<Statement> sorted2 = new ArrayList<Statement>(list2.size());
1101                 sorted1.addAll(list1);
1102                 sorted2.addAll(list2);
1103                 
1104                 int ss1 = 0;
1105                 int ss2 = 0;
1106                 for (int i = 0; i < list1.size(); ) {
1107                         Statement s1 = list1.get(i);
1108                         int same1 = sameRel(list1, i);  
1109                         for (int j = 0; j < list2.size(); j++) {
1110                                 Statement s2 = list2.get(j);
1111                                 if (scomp.compare(s1, s2) == 0) {
1112                                         int same2 = sameRel(list2, j);
1113                                         copy(sorted1,ss1,list1,i,same1);
1114                                         ss1 += same1;
1115                                         copy(sorted2,ss2,list2,j,same2);
1116                                         ss2 += same2;
1117                                         break;
1118                                 }
1119                         }
1120                         i+= same1;
1121                 }
1122                 if (ss1 < sorted1.size()) {
1123                         for (Statement s : list1) {
1124                                 if (!sorted1.contains(s)) {
1125                                         sorted1.set(ss1,s);
1126                                         ss1++;
1127                                 }
1128                         }
1129                 }
1130                 if (ss2 < sorted2.size()) {
1131                         for (Statement s : list2) {
1132                                 if (!sorted2.contains(s)) {
1133                                         sorted2.set(ss2,s);
1134                                         ss2++;
1135                                 }
1136                         }
1137                 }
1138                 
1139                 list1.clear();
1140                 list2.clear();
1141                 list1.addAll(sorted1);
1142                 list2.addAll(sorted2);
1143         }
1144         
1145         public <T> void copy(List<T> to, int toIndex, List<T> from, int fromIndex, int amount) {
1146                 for (int i = 0; i <  amount; i++) {
1147                         to.set(toIndex + i, from.get(fromIndex+ i));
1148                 }
1149         }
1150         
1151         public void sortResource(List<Resource> list1, List<Resource> list2) {
1152                 Collections.sort(list1,rcomp);
1153                 int js = 0;
1154                 for (int i = 0; i < list1.size(); i++) {
1155                         Resource s1 = list1.get(i);
1156                         for (int j = js; j < list2.size(); j++) {
1157                                 Resource s2 = list2.get(j);
1158                                 if (rcomp.compare(s1, s2) == 0) {
1159                                         Resource t = list2.get(js);
1160                                         list2.set(js, s2);
1161                                         list2.set(j, t);
1162                                         break;
1163                                 }
1164                         }
1165                         js++;
1166                 }
1167         }
1168         
1169         private void compareStatements(List<Statement> ss1, List<Statement> ss2, Stack<Resource> objectsLeft, Stack<Resource> objectsRight, Collection<Statement> unreliableLeft, Collection<Statement> unreliableRight) throws DatabaseException {
1170                 sortStatement(ss1, ss2);
1171                 
1172                 int i1 = 0;
1173                 int i2 = 0;
1174                 
1175                 while (true) {
1176                         if (i1 >= ss1.size()) {
1177                                 if (i2 >= ss2.size()) {
1178                                         break;
1179                                 } else {
1180                                         while (i2 < ss2.size()) {
1181                                                 if (DEBUG) System.out.println("Compare Statements addition " + printStatement(g,ss2.get(i2)));
1182                                                 
1183                                                 addAddition(ss2.get(i2));
1184                                                 i2++;
1185                                         }
1186                                         break;
1187                                 }
1188                         } else if (i2 >= ss2.size()) {
1189                                 while (i1 < ss1.size()) {
1190                                         if (DEBUG) System.out.println("Compare Statements deletion " + printStatement(g,ss1.get(i1)));
1191                                         addDeletion(ss1.get(i1));
1192                                         i1++;
1193                                 }
1194                                 break;
1195                         }
1196                         int same1 = sameRel(ss1, i1);
1197                         int same2 = sameRel(ss2, i2);
1198                         int c = rcomp.compare(ss1.get(i1).getPredicate(),ss2.get(i2).getPredicate());
1199                         if (c == 0) {
1200                                 compareStatements(ss1, i1, same1, ss2, i2, same2,objectsLeft,objectsRight,unreliableLeft,unreliableRight);
1201                                 i1+=same1;
1202                                 i2+=same2;
1203                         } else if (c < 0) {
1204                                 for (int i = 0; i < same1; i++) {
1205                                         if (DEBUG) System.out.println("Compare Statements deletion " + printStatement(g,ss1.get(i+i1)));
1206                                         addDeletion(ss1.get(i+i1));
1207                                 }
1208                                 i1 += same1;
1209                         } else {
1210                                 for (int i = 0; i < same2; i++) {
1211                                         if (DEBUG) System.out.println("Compare Statements addition " + printStatement(g,ss2.get(i+i2)));
1212                                         addAddition(ss2.get(i+i2));
1213                                 }
1214                                 
1215                                 i2 += same2;
1216                         }
1217                 }
1218         }
1219         
1220
1221         
1222         private int sameRel(List<Statement> statements, int off) {
1223                 if (statements.size() <= off)
1224                         return 0;
1225                 int same = 1;
1226                 long id = statements.get(off).getPredicate().getResourceId();
1227                 for (int i = off+1; i <statements.size(); i++) {
1228                         if (statements.get(i).getPredicate().getResourceId() == id)
1229                                 same++;
1230                         else 
1231                                 break;
1232                 }
1233                 return same;
1234                 
1235         }
1236
1237         private int compareObject(Resource o1, Resource o2) throws DatabaseException {
1238                 if (o1.equals(o2))
1239                         return -1;
1240                 if (comparableResources.contains(o1, o2))
1241                         return (-1);
1242                 if (comparableResources.containsLeft(o1))
1243                         return Integer.MAX_VALUE;
1244                 if (comparableResources.containsRight(o2))
1245                         return Integer.MAX_VALUE;
1246                 if (nonMatchedLeft.contains(o1))
1247                         return Integer.MAX_VALUE;
1248                 if (nonMatchedRight.contains(o2))
1249                         return Integer.MAX_VALUE;
1250                 return comparator.compare(g, o1, o2);
1251         }
1252         
1253         private void compareStatements(List<Statement> ss1, int off1, int len1, List<Statement> ss2, int off2, int len2, Collection<Resource> objectsLeft, Collection<Resource> objectsRight, Collection<Statement> unreliableLeft, Collection<Statement> unreliableRight) throws DatabaseException {
1254                 boolean[] used1 = new boolean[len1];
1255                 for (int i = 0; i < used1.length; i++) {
1256                         used1[i] = false;
1257                 }
1258                 
1259                 boolean[] used2 = new boolean[len2];
1260                 for (int i = 0; i < used2.length; i++) {
1261                         used2[i] = false;
1262                 }
1263                                 
1264                 // left, right, difference
1265                 List<List<Integer>> differences = new ArrayList<List<Integer>>();
1266                 for (int i1 = off1; i1 < off1 + len1; i1++) {
1267                         Statement s1 = ss1.get(i1);
1268                         List<Integer> diff = new ArrayList<Integer>();
1269                         for (int i2 = off2; i2 < off2 + len2; i2++) {
1270                                 Statement s2 = ss2.get(i2);
1271                                 int d = compareObject(s1.getObject(), s2.getObject());
1272                                 if (d == 0) {
1273                                         for (Resource t : strong) {
1274                                                  if (s1.getPredicate().equals(t) || g.isSubrelationOf(s1.getPredicate(), t)) {
1275                                                          d = 1;
1276                                                          break;
1277                                                  }
1278                                         }
1279                                 }
1280                                 diff.add(d);
1281                         }
1282                         differences.add(diff);
1283                 }
1284                 // difference, left
1285                 MapList<Integer, Integer> priorities = new MapList<Integer, Integer>();
1286                 for (int i = 0; i < differences.size(); i++) {
1287                         List<Integer> list = differences.get(i);
1288                         for (int j = 0; j < list.size(); j++) {
1289                                 priorities.add(list.get(j), i);
1290                         }
1291                 }
1292                 
1293                 Integer[] pris = priorities.getKeys(new Integer[]{});
1294                 Arrays.sort(pris);
1295                 boolean matchFail = priorityMatching(ss1, off1, len1, ss2, off2, len2, pris, differences, priorities, used1, used2, objectsLeft, objectsRight, false);
1296                 if (matchFail) 
1297                         matchFail = priorityMatching(ss1, off1, len1, ss2, off2, len2, pris, differences, priorities, used1, used2, objectsLeft, objectsRight, objectsLeft == null);
1298                 if (unreliableLeft != null) {
1299                         if (matchFail) {
1300                                 // With match failure, statement matching was aborted. We must move all unmatched statements to unreliable.  
1301                                 for (Integer pri : pris) {
1302                                         if (pri == 0 || pri == Integer.MAX_VALUE)
1303                                                 continue;
1304                                         priorityProcessUnmatched(ss1, off1, len1, ss2, off2, len2, differences, priorities, used1, used2, unreliableLeft, unreliableRight, pri);
1305                                 }
1306                         }
1307                         // Zero means unreliable comparison result, move all unmatched to unreliable. 
1308                         if (org.simantics.utils.datastructures.Arrays.contains(pris, 0)) {
1309                                 priorityProcessUnmatched(ss1, off1, len1, ss2, off2, len2, differences, priorities, used1, used2, unreliableLeft, unreliableRight, 0);
1310                         }
1311                 } 
1312 //      Previous version processed 0 priority statements, even when unreliableLeft collection was null.
1313 //              The problem was that property relations were not filtered in process() from "tested" statements.                
1314 //              else {
1315 //                      if (org.simantics.utils.datastructures.Arrays.contains(pris, 0)) {
1316 //                              priorityProcessUnmatched(ss1, off1, len1, ss2, off2, len2, differences, priorities, used1, used2, unreliableLeft, unreliableRight, 0);
1317 //                      }
1318 //              }
1319                 // Report unmatched statements as changes.
1320                 for (int i1 = off1; i1 < off1 + len1; i1++) {
1321                         if (!used1[i1-off1]) {
1322                                 if (DEBUG) System.out.println("Compare Object deletion " + printStatement(g,ss1.get(i1)));
1323                                 addDeletion(ss1.get(i1));
1324                         }
1325                 }
1326                 for (int i2 = off2; i2 < off2 + len2; i2++) {
1327                         if (!used2[i2-off2]) {
1328                                 if (DEBUG) System.out.println("Compare Object addition " + printStatement(g,ss2.get(i2)));
1329                                 addAddition(ss2.get(i2));
1330                         }
1331                 }
1332         }
1333         
1334         /**
1335          * Moves unmatched statements to unreliable collections.
1336          * @param ss1
1337          * @param off1
1338          * @param len1
1339          * @param ss2
1340          * @param off2
1341          * @param len2
1342          * @param differences
1343          * @param priorities
1344          * @param used1
1345          * @param used2
1346          * @param unreliableLeft
1347          * @param unreliableRight
1348          * @param pri
1349          */
1350         private void priorityProcessUnmatched(List<Statement> ss1, int off1, int len1, List<Statement> ss2, int off2, int len2,List<List<Integer>> differences,MapList<Integer, Integer> priorities,boolean used1[],boolean used2[], Collection<Statement> unreliableLeft, Collection<Statement> unreliableRight, int pri) {
1351                 Set<Statement> s1s = new HashSet<Statement>();
1352                 Set<Statement> s2s = new HashSet<Statement>();
1353                 Set<Integer> s1i = new HashSet<Integer>();
1354                 Set<Integer> s2i = new HashSet<Integer>();
1355                 List<Integer> i1s = priorities.getValues(pri);
1356                 for (Integer i1 : i1s) {
1357                         if (used1[i1])
1358                                 continue;
1359                         List<Integer> i2diff = differences.get(i1);
1360                         for (int i2 = 0; i2 < i2diff.size(); i2++) {
1361                                 if (i2diff.get(i2) == pri) {
1362                                         if (used2[i2])
1363                                                 continue;
1364                                         Statement s1  = ss1.get(i1+off1);
1365                                         Statement s2  = ss2.get(i2+off2);
1366                                         s1s.add(s1);
1367                                         s2s.add(s2);
1368                                         s1i.add(i1);
1369                                         s2i.add(i2);
1370                                 }
1371                         }
1372                 }
1373                 if (unreliableLeft != null) {
1374                         unreliableLeft.addAll(s1s);
1375                         unreliableRight.addAll(s2s);
1376                 }
1377                 for (Integer i : s1i)
1378                         used1[i] = true;
1379                 for (Integer i : s2i)
1380                         used2[i] = true;
1381         }
1382         /**
1383          * Matches left and right side statements.
1384          * 
1385          * When there are two or more equally good matching objects, the behaviour depends on force flag.
1386          *   False:  Matching is aborted and matchFail is returned (method return true).
1387          *   True: equally good matches will be paired randomly. Method always returns false.
1388          * @param ss1
1389          * @param off1
1390          * @param len1
1391          * @param ss2
1392          * @param off2
1393          * @param len2
1394          * @param pris
1395          * @param differences
1396          * @param priorities
1397          * @param used1
1398          * @param used2
1399          * @param objectsLeft
1400          * @param objectsRight
1401          * @param force 
1402          * @return
1403          * @throws DatabaseException
1404          */
1405         private boolean priorityMatching(List<Statement> ss1, int off1, int len1, List<Statement> ss2, int off2, int len2, Integer[] pris, List<List<Integer>> differences, MapList<Integer, Integer> priorities, boolean used1[],boolean used2[], Collection<Resource> objectsLeft, Collection<Resource> objectsRight, boolean force) throws DatabaseException {
1406                 boolean matchFail = false;
1407                 for (Integer pri : pris) {
1408                         if (pri == Integer.MAX_VALUE) {
1409
1410                         } else if (pri == 0) {
1411                                 
1412                         } else {
1413                                 List<Integer> i1s = priorities.getValues(pri);
1414                                 
1415                                 for (Integer i1 : i1s) {
1416                                         if (used1[i1])
1417                                                 continue;
1418                                         List<Integer> i2diff = differences.get(i1);
1419                                         List<Integer> matches = new ArrayList<Integer>();
1420                                         for (int i2 = 0; i2 < i2diff.size(); i2++) {
1421                                                 if (i2diff.get(i2) == pri) { 
1422                                                         if (used2[i2])
1423                                                                 continue;
1424                                                         matches.add(i2);
1425                                                 }
1426                                         }
1427                                         if (matches.size() == 1 || (force && matches.size() > 1)) {
1428                                                 int i2 = matches.get(0);
1429                                                 used1[i1] = true;
1430                                                 used2[i2] = true;
1431                                                 Statement s1  = ss1.get(i1+off1);
1432                                                 Statement s2  = ss2.get(i2+off2);
1433                                                 
1434                                                 if (objectsLeft != null) {
1435                                                         objectsLeft.add(s1.getObject());
1436                                                         objectsRight.add(s2.getObject());
1437                                                 } 
1438                                                 addComparable(s1, s2);
1439                                         } else if (matches.size() > 1) {
1440                                                 matchFail = true;
1441                                         }
1442                                 }
1443                                 if (matchFail)
1444                                         break;
1445                         }
1446                 }
1447                 return matchFail;
1448         }
1449
1450         
1451         /**
1452          * compares properties, assumes functional relations
1453          * @param r1
1454          * @param r2
1455          * @throws DatabaseException 
1456          */
1457         private void compareProps(Resource r1, Resource r2) throws DatabaseException {
1458                 if (DEBUG) System.out.println("compareProps " + r1 + " " + NameUtils.getSafeName(g, r1) + " " + r2 + " " + NameUtils.getSafeName(g, r2));
1459                 List<Statement> ss1 = new ArrayList<Statement>();
1460                 List<Statement> ss2 = new ArrayList<Statement>();
1461                 ss1.addAll(g.getStatements(r1, b.HasProperty));
1462                 ss2.addAll(g.getStatements(r2, b.HasProperty));
1463                 ss1 = filterNonTested(ss1);
1464                 ss2 = filterNonTested(ss2);
1465                 sortStatement(ss1, ss2);
1466                 // getStatements(r1, b.HasProperty) returns both instance and asserted statements for the same property relation. 
1467                 ss1 = filterAssertedDuplicates(r1, ss1);
1468                 ss2 = filterAssertedDuplicates(r2, ss2);
1469                 
1470                 int i1 = 0; 
1471                 int i2 = 0;
1472                 
1473                 while (true) {
1474                         if (i1 >= ss1.size()) {
1475                                 if (i2 >= ss2.size())
1476                                         break;
1477                                 else {
1478                                         while (i2 < ss2.size()) {
1479                                                 Statement s = ss2.get(i2);
1480                                                 if (DEBUG) System.out.println("Compare Prop diff2 " + printStatement(g,s));
1481                                                 if (!s.isAsserted(r2))
1482                                                         addAddition(s);
1483                                                 i2++;
1484                                         }
1485                                         break;
1486                                 }
1487                         } else if (i2 >= ss2.size()) {
1488                                 while (i1 < ss1.size()) {
1489                                         Statement s = ss1.get(i1);
1490                                         if (DEBUG) System.out.println("Compare Prop diff1 " + printStatement(g,s));
1491                                         if (!s.isAsserted(r1))
1492                                                 addDeletion(s);
1493                                         i1++;
1494                                 }
1495                                 break;
1496                         }
1497                         Statement s1 = ss1.get(i1);
1498                         Statement s2 = ss2.get(i2);
1499                         if (s1.isAsserted(r1) && s2.isAsserted(r2)) {
1500                                 i1++;
1501                                 i2++;
1502                                 continue;
1503                         }
1504                         int c = scomp.compare(s1, s2);
1505                         switch (c) {
1506                                 case 0:{
1507                                         boolean b1 = g.hasValue(s1.getObject());
1508                                         boolean b2 = g.hasValue(s2.getObject());
1509                                         boolean a1 = s1.isAsserted(r1);
1510                                         boolean a2 = s2.isAsserted(r2);
1511                                         if (b1 == b2) {
1512                                                 if (b1) {
1513                                                         // Literals
1514                                                         boolean eq = compareValue(g,b,s1.getObject(), s2.getObject());
1515                                                         if (!eq) {
1516                                                                 addModification(r1,s1,r2,s2);
1517                                                         }
1518                                                         if (mapLiterals && !a1 && !a2)
1519                                                                 addComparable(s1, s2); 
1520                                                 } else {
1521                                                         // Non literal properties.
1522                                                         int comp = comparator.compare(g, s1.getObject(), s2.getObject());
1523                                                         if (comp == ResourceComparator.NO_MATCH) {
1524                                                                 addModification(r1,s1,r2,s2);
1525                                                         } else if (comp != ResourceComparator.EXACT_MATCH) {
1526                                                                 if (!s1.getObject().equals(s1.getSubject()) && !s2.getObject().equals(s2.getSubject())) {
1527                                                                         if (!a1 && !a2) {
1528                                                                                 // compare props matches objects, so we can call that only for non asserted statements
1529                                                                                 compareProps(s1.getObject(), s2.getObject());
1530                                                                         } else if (a1 && a2) {
1531                                                                                 // TODO : compare asserted statements?
1532                                                                         } else {
1533                                                                                 addModification(r1,s1,r2,s2);
1534                                                                         }
1535                                                                 } else {
1536                                                                         addModification(r1,s1,r2,s2);
1537                                                                 }
1538                                                         } else {
1539                                                                 // Exact match, nothing to do.
1540                                                                 if (!a1 && !a2)
1541                                                                         addComparable(s1, s2);
1542                                                         }
1543                                                 }
1544                                         } else {
1545                                                 addModification(r1,s1,r2,s2);
1546                                         }
1547                                         i1++;
1548                                         i2++;
1549                                         break;
1550                                 }
1551                                 case -1:{
1552                                         if (DEBUG) System.out.println("Compare Prop diff1s " + printStatement(g,s1));
1553                                         // Use modification, since deletions do not support asserted statements
1554                                         addModification(r1,s1,r2,null);
1555                                         //addDeletion(s1);
1556                                         i1++;
1557                                         break;
1558                                 }
1559                                         
1560                                 case 1:{
1561                                         if (DEBUG) System.out.println("Compare Prop diff2s " + printStatement(g,s2));
1562                                         // Use modification, since additions do not support asserted statements
1563                                         addModification(r1,null,r2,s2);
1564                                         //addAddition(s2);
1565                                         i2++;
1566                                         break;
1567                                 }
1568                         }
1569
1570                 }
1571                 
1572                 ss1.clear();
1573                 ss2.clear();
1574                 
1575         }
1576         
1577         public static boolean compareValue(ReadGraph g, Layer0 b, Resource r1, Resource r2) throws DatabaseException {
1578                 Resource t1 = g.getSingleType(r1);
1579                 Resource t2 = g.getSingleType(r2);
1580                 if (!t1.equals(t2))
1581                         return false;
1582                 if (t1.equals(b.Integer)) {
1583                         int v1 = g.getValue(r1, Bindings.INTEGER);
1584                         int v2 = g.getValue(r2, Bindings.INTEGER);
1585                         return v1 == v2;
1586                 } else if (t1.equals(b.Float)) {
1587                         float v1 = g.getValue(r1, Bindings.FLOAT);
1588                         float v2 = g.getValue(r2, Bindings.FLOAT);
1589                         return v1 == v2;
1590                 } else if (t1.equals(b.Double)) {
1591                         double v1 = g.getValue(r1, Bindings.DOUBLE);
1592                         double v2 = g.getValue(r2, Bindings.DOUBLE);
1593                         return v1 == v2;
1594                 } else if (t1.equals(b.String)) {
1595                         String v1 = g.getValue(r1, Bindings.STRING);
1596                         String v2 = g.getValue(r2, Bindings.STRING);
1597                         return v1.equals(v2);
1598                 } else if (t1.equals(b.Boolean)) {
1599                         boolean v1 = g.getValue(r1, Bindings.BOOLEAN);
1600                         boolean v2 = g.getValue(r2, Bindings.BOOLEAN);
1601                         return v1 == v2;
1602                 } else if (t1.equals(b.Byte)) {
1603                         byte v1 = g.getValue(r1, Bindings.BYTE);
1604                         byte v2 = g.getValue(r2, Bindings.BYTE);
1605                         return v1 == v2;
1606                 } else if (t1.equals(b.Long)) {
1607                         long v1 = g.getValue(r1, Bindings.LONG);
1608                         long v2 = g.getValue(r2, Bindings.LONG);
1609                         return v1 == v2;
1610                 } else if (t1.equals(b.IntegerArray)) {
1611                         int[] v1 = g.getValue(r1, Bindings.INT_ARRAY);
1612                         int[] v2 = g.getValue(r2, Bindings.INT_ARRAY);
1613                         return Arrays.equals(v1,v2);
1614                 } else if (t1.equals(b.FloatArray)) {
1615                         float[] v1 = g.getValue(r1, Bindings.FLOAT_ARRAY);
1616                         float[] v2 = g.getValue(r2, Bindings.FLOAT_ARRAY);
1617                         return Arrays.equals(v1,v2);
1618                 } else if (t1.equals(b.DoubleArray)) {
1619                         double[] v1 = g.getValue(r1, Bindings.DOUBLE_ARRAY);
1620                         double[] v2 = g.getValue(r2, Bindings.DOUBLE_ARRAY);
1621                         return Arrays.equals(v1,v2);
1622                 } else if (t1.equals(b.StringArray)) {
1623                         String[] v1 = g.getValue(r1, Bindings.STRING_ARRAY);
1624                         String[] v2 = g.getValue(r2, Bindings.STRING_ARRAY);
1625                         return Arrays.equals(v1,v2);
1626                 } else if (t1.equals(b.BooleanArray)) {
1627                         boolean[] v1 = g.getValue(r1, Bindings.BOOLEAN_ARRAY);
1628                         boolean[] v2 = g.getValue(r2, Bindings.BOOLEAN_ARRAY);
1629                         return Arrays.equals(v1,v2);
1630                 } else if (t1.equals(b.ByteArray)) {
1631                         byte[] v1 = g.getValue(r1, Bindings.BYTE_ARRAY);
1632                         byte[] v2 = g.getValue(r2, Bindings.BYTE_ARRAY);
1633                         return Arrays.equals(v1,v2);
1634                 } else if (t1.equals(b.LongArray)) {
1635                         long[] v1 = g.getValue(r1, Bindings.LONG_ARRAY);
1636                         long[] v2 = g.getValue(r2, Bindings.LONG_ARRAY);
1637                         return Arrays.equals(v1,v2);
1638                 } else {
1639                         Object v1 = g.getValue(r1);
1640                         Object v2 = g.getValue(r2);
1641                         return compareValue(v1, v2);
1642                 }
1643                 
1644         }
1645         
1646         public static boolean compareValue(Object v1, Object v2) {
1647                 if (v1 instanceof Object[] && v2 instanceof Object[])
1648                         return Arrays.deepEquals((Object[])v1, (Object[])v2);
1649                 else if (v1 instanceof int[] && v2 instanceof int[]) 
1650                         return Arrays.equals((int[])v1, (int[])v2);
1651                 else if (v1 instanceof float[] && v2 instanceof float[]) 
1652                         return Arrays.equals((float[])v1, (float[])v2);
1653                 else if (v1 instanceof double[] && v2 instanceof double[]) 
1654                         return Arrays.equals((double[])v1, (double[])v2);
1655                 else if (v1 instanceof long[] && v2 instanceof long[]) 
1656                         return  Arrays.equals((long[])v1, (long[])v2);
1657                 else if (v1 instanceof byte[] && v2 instanceof byte[]) 
1658                         return Arrays.equals((byte[])v1, (byte[])v2);
1659                 else if (v1 instanceof boolean[] && v2 instanceof boolean[]) 
1660                         return Arrays.equals((boolean[])v1, (boolean[])v2);
1661                 else
1662                         return v1.equals(v2);
1663         }
1664
1665         
1666         public class PredicateComparator implements Comparator<Statement> {
1667                 @Override
1668                 public int compare(Statement o1, Statement o2) {
1669                         if (comparableResources.contains(o1.getPredicate(), o2.getPredicate()))
1670                                 return 0;
1671                         if (o1.getPredicate().getResourceId() < o2.getPredicate().getResourceId())
1672                                 return -1;
1673                         if (o1.getPredicate().getResourceId() > o2.getPredicate().getResourceId())
1674                                 return 1;
1675                         return 0;
1676                 }
1677         }
1678         
1679         public class SubjectComparator implements Comparator<Statement> {
1680                 @Override
1681                 public int compare(Statement o1, Statement o2) {
1682                         if (comparableResources.contains(o1.getSubject(), o2.getSubject()))
1683                                 return 0;
1684                         if (o1.getSubject().getResourceId() < o2.getSubject().getResourceId())
1685                                 return -1;
1686                         if (o1.getSubject().getResourceId() > o2.getSubject().getResourceId())
1687                                 return 1;
1688                         return 0;
1689                 }
1690         }
1691         
1692         public class ObjectComparator implements Comparator<Statement> {
1693                 @Override
1694                 public int compare(Statement o1, Statement o2) {
1695                         if (comparableResources.contains(o1.getObject(), o2.getObject()))
1696                                 return 0;
1697                         if (o1.getObject().getResourceId() < o2.getObject().getResourceId())
1698                                 return -1;
1699                         if (o1.getObject().getResourceId() > o2.getObject().getResourceId())
1700                                 return 1;
1701                         return 0;
1702                 }
1703         }
1704         
1705         public static class FullStatementComparator implements Comparator<Statement> {
1706                 @Override
1707                 public int compare(Statement o1, Statement o2) {
1708                         if (o1.getSubject().getResourceId() < o2.getSubject().getResourceId())
1709                                 return -1;
1710                         if (o1.getSubject().getResourceId() > o2.getSubject().getResourceId())
1711                                 return 1;
1712                         if (o1.getPredicate().getResourceId() < o2.getPredicate().getResourceId())
1713                                 return -1;
1714                         if (o1.getPredicate().getResourceId() > o2.getPredicate().getResourceId())
1715                                 return 1;
1716                         if (o1.getObject().getResourceId() < o2.getObject().getResourceId())
1717                                 return -1;
1718                         if (o1.getObject().getResourceId() > o2.getObject().getResourceId())
1719                                 return 1;
1720                         return 0;
1721                 }
1722         }
1723         
1724         public class ResComparator implements Comparator<Resource> {
1725                 @Override
1726                 public int compare(Resource o1, Resource o2) {
1727                         if (comparableResources.contains(o1, o2))
1728                                 return 0;
1729                         if (o1.getResourceId() < o2.getResourceId())
1730                                 return -1;
1731                         if (o1.getResourceId() > o2.getResourceId())
1732                                 return 1;
1733                         return 0;
1734                 }
1735         }
1736
1737 }