org.simantics.interop/src/org/simantics/interop/test/GraphComparator.java

   1 /*******************************************************************************
   2  * Copyright (c) 2007, 2010 Association for Decentralized Information Management
   3  * in Industry THTH ry.
   4  * All rights reserved. This program and the accompanying materials
   5  * are made available under the terms of the Eclipse Public License v1.0
   6  * which accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *     Foster Wheeler Energia Oy - initial API and implementation
  11  *     VTT Technical Research Centre of Finland - Improvements to comparison.
  12  *     Semantum Oy - Improvements to comparison, various bug fixes.
  13  *******************************************************************************/
  14 package org.simantics.interop.test;
  15
  16 import java.util.ArrayList;
  17 import java.util.Arrays;
  18 import java.util.Collection;
  19 import java.util.Collections;
  20 import java.util.Comparator;
  21 import java.util.HashMap;
  22 import java.util.HashSet;
  23 import java.util.List;
  24 import java.util.Map;
  25 import java.util.Map.Entry;
  26 import java.util.Set;
  27 import java.util.Stack;
  28
  29 import org.eclipse.core.runtime.IProgressMonitor;
  30 import org.eclipse.core.runtime.NullProgressMonitor;
  31 import org.simantics.databoard.Bindings;
  32 import org.simantics.db.ReadGraph;
  33 import org.simantics.db.Resource;
  34 import org.simantics.db.Session;
  35 import org.simantics.db.Statement;
  36 import org.simantics.db.common.request.ReadRequest;
  37 import org.simantics.db.common.utils.NameUtils;
  38 import org.simantics.db.exception.DatabaseException;
  39 import org.simantics.db.request.Read;
  40 import org.simantics.interop.test.GraphChanges.Modification;
  41 import org.simantics.layer0.Layer0;
  42 import org.simantics.utils.datastructures.BijectionMap;
  43 import org.simantics.utils.datastructures.MapList;
  44 import org.simantics.utils.datastructures.Pair;
  45
  46 /**
  47  * Compares two subgraphs and reports differences.
  48  *
  49  * Assumes that subgraphs (defined using traverse relations) are not cyclic.
  50  *
  51  * Assumes that properties can be used to identify objects, if relation type is not enough.
  52  *
  53  *
  54  *
  55  * @author Marko Luukkainen <marko.luukkainen@vtt.fi>
  56  *
  57  */
  58 public class GraphComparator {
  59
  60         private static final boolean DEBUG = false;
  61
  62         private Resource r1;
  63         private Resource r2;
  64         private Set<Resource> strong = new HashSet<>();       // List of relations that identify object, if subject is already identified.
  65         private List<Resource> traversed = new ArrayList<>(); // list of relations that are traversed (and tested)
  66         private List<Resource> tested = new ArrayList<>();    // list of relations that are tested, but not traversed
  67         private List<Resource> nonTraversed = new ArrayList<>(); // list of relations that are not traversed
  68         private List<Resource> nonTested = new ArrayList<>(); // list of relations that are not tested
  69
  70         private List<Statement> changes1 = new ArrayList<>();
  71         private List<Statement> changes2 = new ArrayList<>();
  72         private List<Modification> modifications = new ArrayList<>();
  73         private Set<Statement> changes1Set = new HashSet<>();
  74         private Set<Statement> changes2Set = new HashSet<>();
  75         private Set<Modification> modificationsSet = new HashSet<>();
  76
  77         private BijectionMap<Statement, Statement> comparableStatements = new BijectionMap<>();
  78         private BijectionMap<Resource, Resource> comparableResources = new BijectionMap<>();
  79
  80         private Set<Resource> processedResources = new HashSet<Resource>();
  81
  82         private ResourceComparator comparator;
  83
  84         private Comparator<Statement> scomp = new PredicateComparator();
  85         private Comparator<Resource> rcomp = new ResComparator();
  86
  87         private Set<Resource> nonMatchedLeft = new HashSet<Resource>();
  88         private Set<Resource> nonMatchedRight = new HashSet<Resource>();
  89
  90         private long iter = 0;
  91         private long maxIter = -1;
  92
  93         // runtime attributes
  94
  95         private ReadGraph g;
  96         private Layer0 b;
  97
  98         public GraphComparator(Resource r1, Resource r2) {
  99                 this.r1 = r1;
 100                 this.r2 = r2;
 101                 comparator = new TypeComparator();
 102         }
 103
 104         public GraphComparator(Resource r1, Resource r2, ResourceComparator comparator) {
 105                 this.r1 = r1;
 106                 this.r2 = r2;
 107                 this.comparator = comparator;
 108         }
 109
 110         ArrayList<Statement> ss1 = new ArrayList<Statement>();
 111         ArrayList<Statement> ss2 = new ArrayList<Statement>();
 112
 113
 114         public Comparator<Resource> getResourceComparator() {
 115                 return rcomp;
 116         }
 117
 118         public Comparator<Statement> getStatementComparator() {
 119                 return scomp;
 120         }
 121
 122         public Resource getR1() {
 123                 return r1;
 124         }
 125
 126         public Resource getR2() {
 127                 return r2;
 128         }
 129
 130         public void addTraversed(Resource rel) {
 131                 traversed.add(rel);
 132         }
 133
 134         public void addTraversed(Collection<Resource> rels) {
 135                 traversed.addAll(rels);
 136         }
 137
 138         public void addNonTraversed(Resource rel) {
 139                 nonTraversed.add(rel);
 140         }
 141
 142         public void addNonTraversed(Collection<Resource> rels) {
 143                 nonTraversed.addAll(rels);
 144         }
 145
 146         public void addTested(Resource rel) {
 147                 tested.add(rel);
 148         }
 149
 150         public void addTested(Collection<Resource> rels) {
 151                 tested.addAll(rels);
 152         }
 153
 154         public void addNonTested(Resource rel) {
 155                 nonTested.add(rel);
 156         }
 157
 158         public void addNonTested(Collection<Resource> rels) {
 159                 nonTested.addAll(rels);
 160         }
 161
 162         public void addComparableResources(Resource r1, Resource r2) {
 163                 if (DEBUG)
 164                         System.out.println("Preset " + r1 + " = " + r2);
 165                 comparableResources.map(r1, r2);
 166         }
 167
 168         public void addComparableResources(BijectionMap<Resource, Resource> matching) {
 169                 if (DEBUG) {
 170                         for (Entry<Resource, Resource> entry : matching.getEntries())
 171                                 System.out.println("Preset " + entry.getKey() + " = " + entry.getValue());
 172                 }
 173                 comparableResources.addAll(matching);
 174         }
 175
 176         public void addStrong(Resource r) {
 177                 strong.add(r);
 178         }
 179
 180         public void addStrong(Collection<Resource> rels) {
 181                 strong.addAll(rels);
 182         }
 183
 184         public void addNonMatchedLeft(Resource r) {
 185                 nonMatchedLeft.add(r);
 186         }
 187
 188         public void addNonMatchedRight(Resource r) {
 189                 nonMatchedRight.add(r);
 190         }
 191
 192         public long getMaxIter() {
 193                 return maxIter;
 194         }
 195
 196         /**
 197          * Sets maximum iteration done on a single DB transaction. Affects only test(Session) methods.
 198          * Default value is -1, which disables iteration limit.
 199          * Note that using iteration limit causes comparison to repeat some work, thus total execution time will increase.
 200          *
 201          * @param maxIter
 202          */
 203         public void setMaxIter(long maxIter) {
 204                 this.maxIter = maxIter;
 205         }
 206
 207         public void clear() {
 208                 changes1.clear();
 209                 changes1Set.clear();
 210                 changes2.clear();
 211                 changes2Set.clear();
 212                 comparableResources.clear();
 213                 comparableStatements.clear();
 214                 modifications.clear();
 215                 modificationsSet.clear();
 216                 processedResources.clear();
 217         }
 218
 219         public void test(ReadGraph g) throws DatabaseException {
 220                 test(g,null);
 221         }
 222
 223         public void test(ReadGraph g, IProgressMonitor monitor) throws DatabaseException {
 224                 this.g = g;
 225                 this.b = Layer0.getInstance(g);
 226                 comparator.setComparator(this);
 227                 comparator.initialize(g, r1, r2);
 228
 229                 if (monitor == null)
 230                         monitor = new NullProgressMonitor();
 231
 232                 Stack<Resource> objectsLeft = new Stack<Resource>();
 233                 Stack<Resource> objectsRight = new Stack<Resource>();
 234                 objectsLeft.push(r1);
 235                 objectsRight.push(r2);
 236
 237                 Set<Statement> unreliableLeft = new HashSet<Statement>();
 238                 Set<Statement> unreliableRight = new HashSet<Statement>();
 239
 240                 boolean changed = false;
 241                 while (true) {
 242                         if (objectsLeft.isEmpty() && !changed)
 243                                 break;
 244                         if (monitor.isCanceled()) {
 245                                 clear();
 246                                 return;
 247                         }
 248                         monitor.subTask(monitorTaskName(objectsLeft, objectsRight, unreliableLeft, unreliableRight));
 249                         changed = false;
 250
 251                         // process compares objects that are identified and searches for more resources to process.
 252                         changed = compareIter(g, objectsLeft, objectsRight, unreliableLeft, unreliableRight);
 253                         monitor.worked(1);
 254
 255                 }
 256                 for (Statement s : unreliableLeft) {
 257                         if (!comparableStatements.containsLeft(s)) {
 258                                 if (DEBUG) System.out.println("Unreliable Object deletion " + printStatement(g,s));
 259                                 addDeletion(s);
 260                         }
 261                 }
 262                 for (Statement s : unreliableRight) {
 263                         if (!comparableStatements.containsRight(s)) {
 264                                 if (DEBUG) System.out.println("Unreliable Object addition " + printStatement(g,s));
 265                                 addAddition(s);
 266                         }
 267                 }
 268                 monitor.subTask(monitorTaskName(objectsLeft, objectsRight, unreliableLeft, unreliableRight) + " done.");
 269                 //printStats();
 270         }
 271
 272         public void test(Session session) throws DatabaseException {
 273                 test(session, r1, r2, null);
 274         }
 275
 276         public void test(Session session, IProgressMonitor monitor) throws DatabaseException {
 277                 test(session, r1, r2, monitor);
 278         }
 279
 280         public void test(Session session, Resource r1, Resource r2) throws DatabaseException {
 281                 test(session, r1, r2, null);
 282         }
 283
 284         public void test(Session session, Resource r1, Resource r2, IProgressMonitor monitor) throws DatabaseException {
 285
 286                 comparator.setComparator(this);
 287
 288                 if (monitor == null)
 289                         monitor = new NullProgressMonitor();
 290
 291                 session.syncRequest(new ReadRequest() {
 292
 293             @Override
 294             public void run(ReadGraph graph) throws DatabaseException {
 295                 comparator.initialize(graph, r1, r2);
 296             }
 297         });
 298
 299                 addComparable(r1, r2);
 300
 301                 final Stack<Resource> objectsLeft = new Stack<Resource>();
 302                 final Stack<Resource> objectsRight = new Stack<Resource>();
 303                 objectsLeft.push(r1);
 304                 objectsRight.push(r2);
 305
 306                 final Set<Statement> unreliableLeft = new HashSet<Statement>();
 307                 final Set<Statement> unreliableRight = new HashSet<Statement>();
 308
 309                 boolean changed = false;
 310                 while (true) {
 311                         if (objectsLeft.isEmpty() && !changed)
 312                                 break;
 313                         if (monitor.isCanceled()) {
 314                                 clear();
 315                                 return;
 316                         }
 317                         monitor.subTask(monitorTaskName(objectsLeft, objectsRight, unreliableLeft, unreliableRight));
 318                         changed = session.syncRequest(new Read<Boolean>() {
 319
 320                                 @Override
 321                                 public Boolean perform(ReadGraph graph) throws DatabaseException {
 322                                         //System.out.println("Iter " + monitorTaskName(objectsLeft, objectsRight, unreliableLeft, unreliableRight));
 323                                         g = graph;
 324                                         b = Layer0.getInstance(graph);
 325                                         return compareIter(graph, objectsLeft, objectsRight, unreliableLeft, unreliableRight);
 326                                 }
 327                         });
 328                         monitor.worked(1);
 329
 330
 331                 }
 332                 for (Statement s : unreliableLeft) {
 333                         if (!comparableStatements.containsLeft(s))
 334                                 addDeletion(s);
 335                 }
 336                 for (Statement s : unreliableRight) {
 337                         if (!comparableStatements.containsRight(s))
 338                                 addAddition(s);
 339                 }
 340                 unreliableLeft.clear();
 341                 unreliableRight.clear();
 342
 343                 monitor.subTask(monitorTaskName(objectsLeft, objectsRight, unreliableLeft, unreliableRight) + " done.");
 344                 //printStats();
 345         }
 346
 347         private boolean compareIter(ReadGraph graph, Stack<Resource> objectsLeft, Stack<Resource> objectsRight, Set<Statement> unreliableLeft, Set<Statement> unreliableRight) throws DatabaseException {
 348                 // process compares objects that are identified and searches for more resources to process.
 349                 iter = 0;
 350                 boolean c = process(objectsLeft, objectsRight, unreliableLeft, unreliableRight);
 351                 if (objectsLeft.isEmpty()) {
 352                         // process unreliable handles cases where unidentified statements subject and object have been identified
 353                         c |= processUnreliable(unreliableLeft, unreliableRight);
 354                         // process unreliable handles cases where unidentified resources have path of length one to identified resource
 355                         if (!c) {
 356                                 c |= processUnreliable(unreliableLeft, unreliableRight,objectsLeft,objectsRight);
 357                                 if (!c && objectsLeft.isEmpty() && unreliableLeft.size() > 0 && unreliableRight.size() > 0) {
 358                                         c |= processUnreliable2(unreliableLeft, unreliableRight,objectsLeft,objectsRight);
 359                                 }
 360                                 if (!c && objectsLeft.isEmpty() && unreliableLeft.size() > 0 && unreliableRight.size() > 0) {
 361                                         // comparison is ending, but we have still unprocessed unidentified resources left.
 362                                         // These cases have longer path than one to identified objects.
 363                                         c |= processUnreliableDeep(unreliableLeft, unreliableRight, objectsLeft, objectsRight);
 364                                 }
 365                                 if (!c && objectsLeft.isEmpty() && unreliableLeft.size() > 0 && unreliableRight.size() > 0) {
 366                                         // comparison is ending, but we have still unprocessed unidentified resources left.
 367                                         // These cases have longer path than one to identified objects.
 368                                         c |= processUnreliableDeep(unreliableLeft, unreliableRight, objectsLeft, objectsRight);
 369                                 }
 370                         }
 371                 }
 372                 return c;
 373
 374         }
 375
 376         private String monitorTaskName(Stack<Resource> objectsLeft, Stack<Resource> objectsRight, Set<Statement> unreliableLeft, Set<Statement> unreliableRight) {
 377                 int cr = comparableResources.size();
 378                 int ch = Math.max(changes1.size(), changes2.size());
 379                 return "Graph compare " + (cr + ch) +  " / " + (cr+ch+(Math.max(objectsLeft.size(), objectsRight.size())+Math.max(unreliableLeft.size(), unreliableRight.size())));
 380         }
 381
 382         private boolean process(Stack<Resource> objectsLeft, Stack<Resource> objectsRight, Set<Statement> unreliableLeft, Set<Statement> unreliableRight) throws DatabaseException {
 383                 List<Statement> ss1 = new ArrayList<Statement>();
 384                 List<Statement> ss2 = new ArrayList<Statement>();
 385                 boolean didSomething = false;
 386                 while (!objectsLeft.isEmpty()&& (maxIter < 0 || iter < maxIter)) {
 387                         Resource r1 = objectsLeft.pop();
 388                         Resource r2 = objectsRight.pop();
 389
 390                         if (r1.equals(r2))
 391                                 continue;
 392                         if (processedResources.contains(r1))
 393                                 continue;
 394                         processedResources.add(r1);
 395
 396
 397                         if((comparableResources.containsLeft(r1)||comparableResources.containsRight(r2)) && !comparableResources.contains(r1, r2)) {
 398                                 throw new DatabaseException("Comparator error: Trying to map " + r1 + " to " + r2 + " while mappings " + r1 + " to " + comparableResources.getRight(r1) + " and " + comparableResources.getLeft(r2) + " to " + r2 + " exist.");
 399                         }
 400                         addComparable(r1, r2);
 401                         didSomething = true;
 402                         //System.out.println("test " + NameUtils.getSafeName(g, r1) + " " + NameUtils.getSafeName(g, r2));
 403                         compareProps(r1, r2);
 404
 405                         for (Resource rel : tested) {
 406                                 ss1.addAll(g.getStatements(r1, rel));
 407                                 ss2.addAll(g.getStatements(r2, rel));
 408                                 ss1 = filterAsserted(r1, ss1);
 409                                 ss2 = filterAsserted(r2, ss2);
 410                                 ss1 = filterTraversed(ss1);
 411                                 ss2 = filterTraversed(ss2);
 412                                 ss1 = filterNonTested(ss1);
 413                                 ss2 = filterNonTested(ss2);
 414                                 ss1 = filter(Collections.singletonList(b.HasProperty), ss1);
 415                                 ss2 = filter(Collections.singletonList(b.HasProperty), ss2);
 416
 417                                 compareStatements(ss1, ss2, null, null,null,null);
 418                                 ss1.clear();
 419                                 ss2.clear();
 420                         }
 421
 422                         for (Resource rel : traversed) {
 423                                 ss1.addAll(g.getStatements(r1, rel));
 424                                 ss2.addAll(g.getStatements(r2, rel));
 425                                 ss1 = filterAsserted(r1, ss1);
 426                                 ss2 = filterAsserted(r2, ss2);
 427                                 ss1 = filterNonTraversed(ss1);
 428                                 ss2 = filterNonTraversed(ss2);
 429                                 compareStatements(ss1, ss2, objectsLeft, objectsRight,unreliableLeft,unreliableRight);
 430                                 ss1.clear();
 431                                 ss2.clear();
 432
 433                         }
 434                         iter++;
 435                 }
 436                 return didSomething;
 437         }
 438
 439         private boolean processUnreliable(Set<Statement> unreliableLeft, Set<Statement> unreliableRight) throws DatabaseException {
 440                 MapList<Resource,Statement> subjectLeft = new MapList<Resource, Statement>();
 441                 MapList<Resource,Statement> subjectRight = new MapList<Resource, Statement>();
 442                 MapList<Resource,Statement> objectLeft = new MapList<Resource, Statement>();
 443                 MapList<Resource,Statement> objectRight = new MapList<Resource, Statement>();
 444                 boolean didSomething = false;
 445                 for (Statement s : unreliableLeft) {
 446                         subjectLeft.add(s.getSubject(),s);
 447                         objectLeft.add(s.getObject(),s);
 448                 }
 449                 for (Statement s : unreliableRight) {
 450                         subjectRight.add(s.getSubject(),s);
 451                         objectRight.add(s.getObject(),s);
 452                 }
 453
 454                 for (Resource left : subjectLeft.getKeys()) {
 455                         if (maxIter > 0 && iter > maxIter)
 456                                 break;
 457                         Resource right = comparableResources.getRight(left);
 458                         if (right == null)
 459                                 continue;
 460                         for (Statement leftS : subjectLeft.getValues(left)) {
 461                                 Resource leftO = leftS.getObject();
 462                                 if (!unreliableLeft.contains(leftS))
 463                                         continue;
 464                                 Resource rightO = comparableResources.getRight(leftO);
 465                                 if (rightO == null)
 466                                         continue;
 467                                 for (Statement rightS : subjectRight.getValues(right)) {
 468                                         if (!rightS.getObject().equals(rightO))
 469                                                 continue;
 470                                         if (!unreliableRight.contains(rightS))
 471                                                 continue;
 472                                         if (leftS.getPredicate().equals(rightS.getPredicate()) ||
 473                                                 comparableResources.contains(leftS.getPredicate(), rightS.getPredicate())) {
 474                                                 unreliableLeft.remove(leftS);
 475                                                 unreliableRight.remove(rightS);
 476                                                 addComparable(leftS, rightS);
 477                                                 didSomething = true;
 478                                                 iter++;
 479                                         }
 480                                 }
 481                         }
 482                 }
 483                 return didSomething;
 484         }
 485
 486         private boolean processUnreliable(Set<Statement> unreliableLeft, Set<Statement> unreliableRight, Stack<Resource> objectsLeft, Stack<Resource> objectsRight) throws DatabaseException {
 487                 MapList<Resource,Statement> subjectLeft = new MapList<Resource, Statement>();
 488                 MapList<Resource,Statement> subjectRight = new MapList<Resource, Statement>();
 489                 MapList<Resource,Statement> objectLeft = new MapList<Resource, Statement>();
 490                 MapList<Resource,Statement> objectRight = new MapList<Resource, Statement>();
 491                 boolean didSomething = false;
 492                 for (Statement s : unreliableLeft) {
 493                         subjectLeft.add(s.getSubject(),s);
 494                         objectLeft.add(s.getObject(),s);
 495                 }
 496                 for (Statement s : unreliableRight) {
 497                         subjectRight.add(s.getSubject(),s);
 498                         objectRight.add(s.getObject(),s);
 499                 }
 500                 for (Resource ol : objectLeft.getKeys()) {
 501                         if (maxIter > 0 && iter > maxIter)
 502                                 break;
 503                         // all statements to the left side object
 504                         List<Statement> left = objectLeft.getValues(ol);
 505                         // all subjects that have statements to the left side object (ol)
 506                         Set<Resource> sLeft = new HashSet<Resource>();
 507                         // all matching subjects on the right side
 508                         Set<Resource> sRight = new HashSet<Resource>();
 509                         for (Statement s : left) {
 510                                 sLeft.add(s.getSubject());
 511                                 sRight.add(comparableResources.getRight(s.getSubject()));
 512                         }
 513
 514                         // check if object left can be reliably identified by available statements
 515                         // if there are any objects on the left side with similar statements, object left cannot be mapped.
 516                         boolean hasSimilar = false;
 517                         MapList<Resource, Statement> comparableOLeft = new MapList<Resource, Statement>();
 518                         for (Resource sl : sLeft) {
 519                                 for (Statement s : subjectLeft.getValues(sl)) {
 520                                         if (!s.getObject().equals(ol)) {
 521                                                 comparableOLeft.add(s.getObject(),s);
 522                                         }
 523                                 }
 524                         }
 525
 526                         compareStatements(ss1, ss2, objectsLeft, objectsRight,unreliableLeft,unreliableRight);
 527
 528                         for (Resource similarOl : comparableOLeft.getKeys()) {
 529                                 List<Statement> similarLeft = comparableOLeft.getValues(similarOl);
 530                                 if (similarLeft.size() == left.size()) {
 531                                         boolean useL[] = new boolean[left.size()];
 532                                         boolean useSL[] = new boolean[left.size()];
 533                                         for (int i = 0; i < left.size(); i++) {
 534                                                 useL[i] = false;
 535                                                 useSL[i] = false;
 536                                         }
 537                                         for (int i = 0; i < left.size(); i++) {
 538                                                 for (int j = 0; j < left.size(); j++) {
 539                                                         if (useSL[j])
 540                                                                 continue;
 541                                                         // compare predicates
 542                                                         Resource pl = left.get(i).getPredicate();
 543                                                         Resource psl = similarLeft.get(j).getPredicate();
 544                                                         if (pl.equals(psl)) {
 545                                                                 // compare objects (unreliable result is interpreted as positive match)
 546
 547                                                                 int comp = comparator.compare(g, left.get(i).getObject(), similarLeft.get(j).getObject(), true);
 548                                                                 if (comp >= 0 && comp < ResourceComparator.NO_MATCH) {
 549                                                                         useL[i] = true;
 550                                                                         useSL[j] = true;
 551                                                                         break;
 552                                                                 }
 553                                                         }
 554                                                 }
 555                                         }
 556                                         boolean diff = false;
 557                                         for (int i = 0; i < left.size(); i++) {
 558                                                 if (!useL[i] || !useSL[i]) {
 559                                                         diff = true;
 560                                                 }
 561                                         }
 562                                         if (!diff) {
 563                                                 hasSimilar = true;
 564                                                 break;
 565                                         }
 566                                 }
 567                         }
 568
 569                         if (hasSimilar)
 570                                 continue;
 571
 572
 573                         // all objects that subjects on the right side point to. Object left has its matching resource among these, if it has matching resource
 574                         MapList<Resource,Statement> possibleOR = new MapList<Resource, Statement>();
 575                         for (Resource sr : sRight) {
 576                                 for (Statement s : subjectRight.getValues(sr))
 577                                         possibleOR.add(s.getObject(),s);
 578                         }
 579
 580                         // filter possible right side objects to those that have same amount of statements as the left side object
 581                         for (Resource or : possibleOR.getKeys().toArray(new Resource[possibleOR.getKeys().size()])) {
 582                                 List<Statement> right = possibleOR.getValues(or);
 583                                 if (right.size() != left.size())
 584                                         possibleOR.remove(or);
 585
 586                         }
 587
 588                         // check for matching statements (comparable subjects, matching predicates)
 589                         MapList<Resource,Statement> matchingOR = new MapList<Resource, Statement>(); // list of objects that have matching statements
 590                         Map<Resource,Pair<int[], int[]>> matchingStatements = new HashMap<Resource, Pair<int[], int[]>>(); // matching statements
 591                         for (Resource or : possibleOR.getKeys()) {
 592                                 List<Statement> right = possibleOR.getValues(or);
 593                                 int iLeft[] = new int[left.size()];
 594                                 int iRight[] = new int[right.size()];
 595
 596                                 for (int i = 0; i < left.size(); i++) {
 597                                         iLeft[i] = -1;
 598                                         iRight[i] = -1;
 599                                 }
 600
 601                                 for (int l = 0; l < left.size(); l++) {
 602                                         Statement ls = left.get(l);
 603                                         for (int r = 0; r < right.size(); r++) {
 604                                                 if (iRight[r] >= 0)
 605                                                         continue;
 606                                                 Statement rs = right.get(r);
 607                                                 if (!comparableResources.contains(ls.getSubject(), rs.getSubject()))
 608                                                         continue;
 609                                                 if ((comparableResources.containsLeft(ls.getObject()) || comparableResources.containsRight(rs.getObject())) && !comparableResources.contains(ls.getObject(), rs.getObject()))
 610                                                         continue;
 611                                                 if (rcomp.compare(ls.getPredicate(),rs.getPredicate()) == 0) {
 612                                                         // compare objects (unreliable result is not accepted)
 613                                                         int comp = comparator.compare(g, ls.getObject(), rs.getObject());
 614                                                         if (comp > 0 && comp < Integer.MAX_VALUE) {
 615                                                                 iLeft[l] = r;
 616                                                                 iRight[r] = l;
 617                                                                 break;
 618                                                         }
 619                                                         break;
 620                                                 }
 621                                         }
 622
 623                                 }
 624                                 boolean success = true;
 625                                 for (int i = 0; i < left.size(); i++) {
 626                                         if (iLeft[i] < 0) {
 627                                                 success = false;
 628                                                 break;
 629                                         }
 630                                         if (iRight[i] < 0) {
 631                                                 success = false;
 632                                                 break;
 633                                         }
 634
 635                                 }
 636                                 if (success) {
 637                                         for (Statement s : right)
 638                                                 matchingOR.add(or,s);
 639                                         matchingStatements.put(or, new Pair<int[], int[]>(iLeft, iRight));
 640                                 }
 641                         }
 642                         // if there is only one matching right side object, we have found a match
 643                         if (matchingOR.getKeySize() == 1) {
 644                                 Resource or = matchingOR.getKeys().iterator().next();
 645                                 List<Statement> right = matchingOR.getValues(or);
 646                                 Pair<int[], int[]> indices = matchingStatements.get(or);
 647
 648                                 objectsLeft.add(ol);
 649                                 objectsRight.add(or);
 650                                 addComparable(ol, or);
 651                                 didSomething = true;
 652                                 for (int l = 0; l < left.size(); l++) {
 653                                         int r = indices.first[l];
 654                                         Statement sl = left.get(l);
 655                                         Statement sr = right.get(r);
 656                                         addComparable(sl, sr);
 657                                         unreliableLeft.remove(sl);
 658                                         unreliableRight.remove(sr);
 659                                 }
 660                                 iter++;
 661
 662                         } else {
 663
 664                         }
 665
 666                 }
 667                 return didSomething;
 668
 669
 670         }
 671
 672         private boolean processUnreliable2(Set<Statement> unreliableLeft, Set<Statement> unreliableRight, Stack<Resource> objectsLeft, Stack<Resource> objectsRight) throws DatabaseException {
 673                 MapList<Resource,Statement> subjectLeft = new MapList<Resource, Statement>();
 674                 MapList<Resource,Statement> subjectRight = new MapList<Resource, Statement>();
 675                 MapList<Resource,Statement> objectLeft = new MapList<Resource, Statement>();
 676                 MapList<Resource,Statement> objectRight = new MapList<Resource, Statement>();
 677                 boolean didSomething = false;
 678                 for (Statement s : unreliableLeft) {
 679                         subjectLeft.add(s.getSubject(),s);
 680                         objectLeft.add(s.getObject(),s);
 681                 }
 682                 for (Statement s : unreliableRight) {
 683                         subjectRight.add(s.getSubject(),s);
 684                         objectRight.add(s.getObject(),s);
 685                 }
 686                 for (Resource ol : objectLeft.getKeys()) {
 687                         if (maxIter > 0 && iter > maxIter)
 688                                 break;
 689                         // all statements to the left side object
 690                         List<Statement> left = objectLeft.getValues(ol);
 691                         // all subjects that have statements to the left side object (ol)
 692                         Set<Resource> sLeft = new HashSet<Resource>();
 693                         // all matching subjects on the right side
 694                         Set<Resource> sRight = new HashSet<Resource>();
 695                         for (Statement s : left) {
 696                                 sLeft.add(s.getSubject());
 697                                 sRight.add(comparableResources.getRight(s.getSubject()));
 698                         }
 699
 700                         if (sLeft.size() == 1 && sRight.size() == 1) {
 701                                 List<Statement> ss1 = new ArrayList<Statement>(subjectLeft.getValues(sLeft.iterator().next()));
 702                                 List<Statement> ss2 = new ArrayList<Statement>(subjectRight.getValues(sRight.iterator().next()));
 703
 704                                 int count = comparableStatements.size();
 705                                 compareStatements(ss1, ss2, objectsLeft, objectsRight,unreliableLeft,unreliableRight);
 706                                 if (comparableStatements.size() > count) {
 707                                         didSomething = true;
 708                                         for (Entry<Statement, Statement> entry : comparableStatements.getEntries()) {
 709                                                 unreliableLeft.remove(entry.getKey());
 710                                                 unreliableRight.remove(entry.getValue());
 711                                                 iter++;
 712                                         }
 713                                 }
 714                         }
 715                 }
 716                 return didSomething;
 717         }
 718
 719         private boolean processUnreliableDeep(Set<Statement> unreliableLeft, Set<Statement> unreliableRight, Stack<Resource> objectsLeft, Stack<Resource> objectsRight) throws DatabaseException {
 720                 MapList<Resource,Statement> subjectLeft = new MapList<Resource, Statement>();
 721                 MapList<Resource,Statement> subjectRight = new MapList<Resource, Statement>();
 722                 MapList<Resource,Statement> objectLeft = new MapList<Resource, Statement>();
 723                 MapList<Resource,Statement> objectRight = new MapList<Resource, Statement>();
 724                 boolean didSomething = false;
 725                 for (Statement s : unreliableLeft) {
 726                         subjectLeft.add(s.getSubject(),s);
 727                         objectLeft.add(s.getObject(),s);
 728                 }
 729                 for (Statement s : unreliableRight) {
 730                         subjectRight.add(s.getSubject(),s);
 731                         objectRight.add(s.getObject(),s);
 732                 }
 733                 for (Resource ol : objectLeft.getKeys()) {
 734                         if (maxIter > 0 && iter > maxIter)
 735                                 break;
 736                         Set<Path> pathsLeft = new HashSet<Path>();
 737                         for (Resource rel : traversed) {
 738                                 pathsLeft.addAll(Path.create(g.getStatements(ol, rel)));
 739                         }
 740                         while (true) {
 741                                 expand(pathsLeft);
 742                                 if (pathsLeft.size() == 0)
 743                                         break;
 744                                 Collection<Path> endPaths = new ArrayList<Path>(1);
 745                                 for (Path p : pathsLeft) {
 746                                         if (comparableResources.containsLeft(p.getEnd())) {
 747                                                 endPaths.add(p);
 748                                         }
 749                                 }
 750                                 if (endPaths.size() > 0) {
 751                                         pathsLeft.clear();
 752                                         pathsLeft.addAll(endPaths);
 753                                         break;
 754                                 }
 755                         }
 756                         if (pathsLeft.size() > 0) {
 757                                 Resource sl = objectLeft.getValues(ol).get(0).getSubject();
 758                                 Resource sr = comparableResources.getRight(sl);
 759                                 Collection<Resource> possibleOR = new ArrayList<Resource>();
 760                                 for (Statement s : subjectRight.getValues(sr)) {
 761                                         possibleOR.add(s.getObject());
 762                                 }
 763                                 Map<Resource,Set<Path>> matchingPaths = new HashMap<Resource, Set<Path>>();
 764                                 for (Resource or : possibleOR) {
 765                                         Set<Path> possiblePathsRight = new HashSet<Path>();
 766                                         for (Path leftPath : pathsLeft) {
 767                                                 possiblePathsRight.addAll(findComparableRight(leftPath, or));
 768                                         }
 769                                         if (hasMatchingPaths(pathsLeft, possiblePathsRight)) {
 770                                                 matchingPaths.put(or, possiblePathsRight);
 771                                         }
 772                                 }
 773                                 if (matchingPaths.size() > 0) {
 774                                         if (matchingPaths.size() == 1) {
 775                                                 didSomething = true;
 776                                                 Resource or = matchingPaths.keySet().iterator().next();
 777
 778                                                 objectsLeft.add(ol);
 779                                                 objectsRight.add(or);
 780                                                 addComparable(ol, or);
 781                                                 Collection<Statement> statementsLeft = objectLeft.getValues(ol);
 782                                                 Collection<Statement> statementsRight = objectRight.getValues(or);
 783                                                 unreliableLeft.removeAll(statementsLeft);
 784                                                 unreliableRight.removeAll(statementsRight);
 785                                                 BijectionMap<Path,Path> map = getMatchingPaths(pathsLeft, matchingPaths.get(or));
 786                                                 for (Path left : map.getLeftSet()) {
 787                                                         Path right = map.getRight(left);
 788                                                         for (int i = 0; i < left.getLength(); i++) {
 789                                                                 addComparable(left.getStatements().get(i),right.getStatements().get(i));
 790                                                                 iter++;
 791                                                         }
 792                                                 }
 793                                                 iter++;
 794                                         }
 795                                 }
 796                         }
 797
 798                 }
 799                 return didSomething;
 800
 801         }
 802
 803         private boolean hasMatchingPaths(Set<Path> leftPaths, Set<Path> rightPaths) throws DatabaseException {
 804                 if (leftPaths.size() != rightPaths.size())
 805                         return false;
 806                 BijectionMap<Path,Path> map = getMatchingPaths(leftPaths, rightPaths);
 807                 return map.size() == leftPaths.size();
 808         }
 809
 810         private BijectionMap<Path,Path> getMatchingPaths(Set<Path> leftPaths, Set<Path> rightPaths) throws DatabaseException {
 811                 BijectionMap<Path,Path> map = new BijectionMap<Path, Path>();
 812                 for (Path leftPath : leftPaths) {
 813                         for (Path rightPath : rightPaths) {
 814                                 if (map.containsRight(rightPath))
 815                                         continue;
 816                                 if (leftPath.getLength() != rightPath.getLength())
 817                                         continue;
 818                                 if (comparableResources.contains(leftPath.getEnd(), rightPath.getEnd())) {
 819                                         boolean match = true;
 820                                         for (int i = 0; i < leftPath.getLength(); i++) {
 821                                                 Statement sl = leftPath.getStatements().get(i);
 822                                                 Statement sr = rightPath.getStatements().get(i);
 823                                                 if (!sl.getPredicate().equals(sr.getPredicate()) && !comparableResources.contains(sl.getPredicate(), sr.getPredicate())) {
 824                                                         match = false;
 825                                                         break;
 826                                                 }
 827                                                 if ((getComparableResources().containsLeft(sl.getObject()) || getComparableResources().containsRight(sr.getObject())) && !getComparableResources().contains(sl.getObject(), sr.getObject())) {
 828                                                         match = false;
 829                                                         break;
 830                                                 }
 831                                                 if (comparator.compare(g, sl.getObject(), sr.getObject()) == ResourceComparator.NO_MATCH) {
 832                                                         match = false;
 833                                                         break;
 834                                                 }
 835                                         }
 836                                         if (match) {
 837                                                 map.map(leftPath, rightPath);
 838                                                 break;
 839                                         }
 840                                 }
 841                         }
 842                 }
 843                 return map;
 844         }
 845
 846         private void expand(Set<Path> paths) throws DatabaseException {
 847                 Set<Path> stepPathsLeft = new HashSet<Path>();
 848                 if (paths.size() == 0)
 849                         return;
 850                 int length = paths.iterator().next().getLength() + 1;
 851                 for (Path p : paths) {
 852                         for (Resource rel : traversed) {
 853                                 stepPathsLeft.addAll(Path.expand(p,g.getStatements(p.getEnd(), rel)));
 854                         }
 855                 }
 856                 paths.clear();
 857                 for (Path p : stepPathsLeft) {
 858                         if (p.getLength() == length)
 859                                 paths.add(p);
 860                 }
 861         }
 862
 863         private Collection<Path> findComparableRight(Path leftPath, Resource beginRight) throws DatabaseException {
 864                 Set<Path> rightPaths = new HashSet<Path>();
 865                 rightPaths.addAll(Path.create(g.getStatements(beginRight, getRight(leftPath.getStatements().get(0).getPredicate()))));
 866                 for (int i = 1; i < leftPath.getLength(); i++) {
 867                         if (rightPaths.size() == 0)
 868                                 return rightPaths;
 869                         Set<Path> stepPaths = new HashSet<Path>();
 870                         for (Path p : rightPaths) {
 871                                 stepPaths.addAll(Path.expand(p, g.getStatements(p.getEnd(), getRight(leftPath.getStatements().get(i).getPredicate()))));
 872                         }
 873                         rightPaths.clear();
 874                         for (Path p : stepPaths)
 875                                 if (p.getLength() == i+1)
 876                                         rightPaths.add(p);
 877                 }
 878                 return rightPaths;
 879
 880         }
 881
 882         private Resource getRight(Resource r) {
 883                 if (comparableResources.containsLeft(r))
 884                         return comparableResources.getRight(r);
 885                 return r;
 886         }
 887
 888
 889
 890         public BijectionMap<Statement, Statement> getComparableStatements() {
 891                 return comparableStatements;
 892         }
 893
 894         public BijectionMap<Resource, Resource> getComparableResources() {
 895                 return comparableResources;
 896         }
 897
 898         public GraphChanges getChanges() {
 899                 return new GraphChanges(r1,r2,changes1,changes2,modifications,comparableResources);
 900         }
 901
 902         public List<Statement> getChanges1() {
 903                 return changes1;
 904         }
 905
 906         public List<Statement> getChanges2() {
 907                 return changes2;
 908         }
 909
 910         private void addComparable(Statement left, Statement right) throws DatabaseException {
 911                 addComparable(left.getObject(), right.getObject());
 912                 comparableStatements.map(left, right);
 913         }
 914
 915         private void addComparable(Resource left, Resource right) throws DatabaseException {
 916                 if(!comparableResources.contains(left, right)) {
 917                         if (comparableResources.containsLeft(left)||comparableResources.containsRight(right)) {
 918                                 throw new DatabaseException("Comparator error: Trying to map " + left + " to " + right + " while mappings " + left + " to " + comparableResources.getRight(left) + " and " + comparableResources.getLeft(right) + " to " + right + " exist.");
 919                         } else {
 920                                 if (DEBUG) System.out.println(left + " = " + right);
 921                                 comparableResources.map(left, right);
 922
 923 //                              if (comparableResources.size() % 1000 == 0)
 924 //                                      printStats();
 925                         }
 926                 }
 927
 928         }
 929
 930         public void printStats() {
 931                 System.out.println("Comp " + comparableResources.size() + " L " + changes1.size() + " R " + changes2.size() + " M " + modifications.size() + " P " + processedResources.size());
 932         }
 933
 934         public List<Statement> filterAsserted(Resource r, Collection<Statement> in) throws DatabaseException {
 935                 List<Statement> out = new ArrayList<Statement>();
 936                 for (Statement s : in) {
 937                         if (!s.isAsserted(r))
 938                                 out.add(s);
 939
 940                 }
 941                 return out;
 942         }
 943
 944         public List<Statement> filterAssertedDuplicates(Resource r, List<Statement> in) throws DatabaseException {
 945                 List<Statement> out = new ArrayList<Statement>();
 946                 for (int i = 0; i < in.size(); i++) {
 947                         Statement s = in.get(i);
 948                         if (!s.isAsserted(r))
 949                                 out.add(s);
 950                         else {
 951                                 boolean has = false;
 952                                 if (i > 0 && in.get(i-1).getPredicate().equals(s.getPredicate()))
 953                                         has = true;
 954                                 else if (i < in.size()-1 && in.get(i+1).getPredicate().equals(s.getPredicate()))
 955                                         has = true;
 956                                 if (!has)
 957                                         out.add(s);
 958                         }
 959
 960                 }
 961                 return out;
 962         }
 963
 964
 965
 966         private String printStatement(ReadGraph graph, Statement s) throws DatabaseException {
 967                 return NameUtils.getSafeName(graph, s.getSubject()) + " " + NameUtils.getSafeName(graph, s.getPredicate()) + " " + NameUtils.getSafeName(graph, s.getObject());
 968         }
 969
 970         private List<Statement> filterTraversed(List<Statement> in) throws DatabaseException {
 971                 return filter(traversed, in);
 972         }
 973
 974         private List<Statement> filterNonTested(List<Statement> in) throws DatabaseException {
 975                 return filter(nonTested, in);
 976         }
 977
 978         private List<Statement> filterNonTraversed(List<Statement> in) throws DatabaseException {
 979                 return filter(nonTraversed, in);
 980         }
 981
 982         private List<Statement> filter(Collection<Resource> toFilter, List<Statement> in) throws DatabaseException {
 983                 if (toFilter.size() == 0)
 984                         return in;
 985                 List<Statement> out = new ArrayList<Statement>();
 986                 for (Statement s : in) {
 987                         boolean usable = true;
 988                         for (Resource r : toFilter) {
 989                                 if (g.isSubrelationOf(s.getPredicate(),r)) {
 990                                         usable = false;
 991                                         break;
 992                                 }
 993                         }
 994                         if (usable) {
 995                                 out.add(s);
 996                         }
 997
 998                 }
 999                 return out;
1000         }
1001
1002
1003         private void addDeletion(Statement s) {
1004                 if (!changes1Set.contains(s)) {
1005                         if (DEBUG) System.out.println("- " +s.getSubject() + " " + s.getPredicate() + " " + s.getObject()) ;
1006                         changes1Set.add(s);
1007                         changes1.add(s);
1008                 }
1009         }
1010
1011         private void addAddition(Statement s) {
1012                 if (!changes2Set.contains(s)) {
1013                         changes2Set.add(s);
1014                         if (DEBUG) System.out.println("+ " +s.getSubject() + " " + s.getPredicate() + " " + s.getObject()) ;
1015                         changes2.add(s);
1016                 }
1017         }
1018
1019         private void addModification(Resource left, Statement leftstm, Resource right, Statement rightstm) {
1020                 Modification mod = new Modification(left, right, leftstm, rightstm);
1021                 if (!modificationsSet.contains(mod)) {
1022                         modificationsSet.add(mod);
1023                         modifications.add(mod);
1024                 }
1025         }
1026
1027         public void sortStatement(List<Statement> list1, List<Statement> list2) {
1028                 sortStatement(list1, list2, scomp);
1029         }
1030
1031         public void sortStatement(List<Statement> list1, List<Statement> list2, Comparator<Statement> scomp) {
1032                 Collections.sort(list1,scomp);
1033                 Collections.sort(list2,scomp);
1034
1035                 List<Statement> sorted1 = new ArrayList<Statement>(list1.size());
1036                 List<Statement> sorted2 = new ArrayList<Statement>(list2.size());
1037                 sorted1.addAll(list1);
1038                 sorted2.addAll(list2);
1039
1040                 int ss1 = 0;
1041                 int ss2 = 0;
1042                 for (int i = 0; i < list1.size(); ) {
1043                         Statement s1 = list1.get(i);
1044                         int same1 = sameRel(list1, i);
1045                         for (int j = 0; j < list2.size(); j++) {
1046                                 Statement s2 = list2.get(j);
1047                                 if (scomp.compare(s1, s2) == 0) {
1048                                         int same2 = sameRel(list2, j);
1049                                         copy(sorted1,ss1,list1,i,same1);
1050                                         ss1 += same1;
1051                                         copy(sorted2,ss2,list2,j,same2);
1052                                         ss2 += same2;
1053                                         break;
1054                                 }
1055                         }
1056                         i+= same1;
1057                 }
1058                 if (ss1 < sorted1.size()) {
1059                         for (Statement s : list1) {
1060                                 if (!sorted1.contains(s)) {
1061                                         sorted1.set(ss1,s);
1062                                         ss1++;
1063                                 }
1064                         }
1065                 }
1066                 if (ss2 < sorted2.size()) {
1067                         for (Statement s : list2) {
1068                                 if (!sorted2.contains(s)) {
1069                                         sorted2.set(ss2,s);
1070                                         ss2++;
1071                                 }
1072                         }
1073                 }
1074
1075                 list1.clear();
1076                 list2.clear();
1077                 list1.addAll(sorted1);
1078                 list2.addAll(sorted2);
1079         }
1080
1081         public <T> void copy(List<T> to, int toIndex, List<T> from, int fromIndex, int amount) {
1082                 for (int i = 0; i <  amount; i++) {
1083                         to.set(toIndex + i, from.get(fromIndex+ i));
1084                 }
1085         }
1086
1087         public void sortResource(List<Resource> list1, List<Resource> list2) {
1088                 Collections.sort(list1,rcomp);
1089                 int js = 0;
1090                 for (int i = 0; i < list1.size(); i++) {
1091                         Resource s1 = list1.get(i);
1092                         for (int j = js; j < list2.size(); j++) {
1093                                 Resource s2 = list2.get(j);
1094                                 if (rcomp.compare(s1, s2) == 0) {
1095                                         Resource t = list2.get(js);
1096                                         list2.set(js, s2);
1097                                         list2.set(j, t);
1098                                         break;
1099                                 }
1100                         }
1101                         js++;
1102                 }
1103         }
1104
1105         private void compareStatements(List<Statement> ss1, List<Statement> ss2, Stack<Resource> objectsLeft, Stack<Resource> objectsRight, Collection<Statement> unreliableLeft, Collection<Statement> unreliableRight) throws DatabaseException {
1106                 sortStatement(ss1, ss2);
1107
1108                 int i1 = 0;
1109                 int i2 = 0;
1110
1111                 while (true) {
1112                         if (i1 >= ss1.size()) {
1113                                 if (i2 >= ss2.size()) {
1114                                         break;
1115                                 } else {
1116                                         while (i2 < ss2.size()) {
1117                                                 if (DEBUG) System.out.println("Compare Statements addition " + printStatement(g,ss2.get(i2)));
1118
1119                                                 addAddition(ss2.get(i2));
1120                                                 i2++;
1121                                         }
1122                                         break;
1123                                 }
1124                         } else if (i2 >= ss2.size()) {
1125                                 while (i1 < ss1.size()) {
1126                                         if (DEBUG) System.out.println("Compare Statements deletion " + printStatement(g,ss1.get(i1)));
1127                                         addDeletion(ss1.get(i1));
1128                                         i1++;
1129                                 }
1130                                 break;
1131                         }
1132                         int same1 = sameRel(ss1, i1);
1133                         int same2 = sameRel(ss2, i2);
1134                         int c = rcomp.compare(ss1.get(i1).getPredicate(),ss2.get(i2).getPredicate());
1135                         if (c == 0) {
1136                                 compareStatements(ss1, i1, same1, ss2, i2, same2,objectsLeft,objectsRight,unreliableLeft,unreliableRight);
1137                                 i1+=same1;
1138                                 i2+=same2;
1139                         } else if (c < 0) {
1140                                 for (int i = 0; i < same1; i++) {
1141                                         if (DEBUG) System.out.println("Compare Statements deletion " + printStatement(g,ss1.get(i+i1)));
1142                                         addDeletion(ss1.get(i+i1));
1143                                 }
1144                                 i1 += same1;
1145                         } else {
1146                                 for (int i = 0; i < same2; i++) {
1147                                         if (DEBUG) System.out.println("Compare Statements addition " + printStatement(g,ss2.get(i+i2)));
1148                                         addAddition(ss2.get(i+i2));
1149                                 }
1150
1151                                 i2 += same2;
1152                         }
1153                 }
1154         }
1155
1156
1157
1158         private int sameRel(List<Statement> statements, int off) {
1159                 if (statements.size() <= off)
1160                         return 0;
1161                 int same = 1;
1162                 long id = statements.get(off).getPredicate().getResourceId();
1163                 for (int i = off+1; i <statements.size(); i++) {
1164                         if (statements.get(i).getPredicate().getResourceId() == id)
1165                                 same++;
1166                         else
1167                                 break;
1168                 }
1169                 return same;
1170
1171         }
1172
1173         private int compareObject(Resource o1, Resource o2) throws DatabaseException {
1174                 if (o1.equals(o2))
1175                         return -1;
1176                 if (comparableResources.contains(o1, o2))
1177                         return (-1);
1178                 if (comparableResources.containsLeft(o1))
1179                         return Integer.MAX_VALUE;
1180                 if (comparableResources.containsRight(o2))
1181                         return Integer.MAX_VALUE;
1182                 if (nonMatchedLeft.contains(o1))
1183                         return Integer.MAX_VALUE;
1184                 if (nonMatchedRight.contains(o2))
1185                         return Integer.MAX_VALUE;
1186                 return comparator.compare(g, o1, o2);
1187         }
1188
1189         private void compareStatements(List<Statement> ss1, int off1, int len1, List<Statement> ss2, int off2, int len2, Collection<Resource> objectsLeft, Collection<Resource> objectsRight, Collection<Statement> unreliableLeft, Collection<Statement> unreliableRight) throws DatabaseException {
1190                 boolean[] used1 = new boolean[len1];
1191                 for (int i = 0; i < used1.length; i++) {
1192                         used1[i] = false;
1193                 }
1194
1195                 boolean[] used2 = new boolean[len2];
1196                 for (int i = 0; i < used2.length; i++) {
1197                         used2[i] = false;
1198                 }
1199
1200                 // left, right, difference
1201                 List<List<Integer>> differences = new ArrayList<List<Integer>>();
1202                 for (int i1 = off1; i1 < off1 + len1; i1++) {
1203                         Statement s1 = ss1.get(i1);
1204                         List<Integer> diff = new ArrayList<Integer>();
1205                         for (int i2 = off2; i2 < off2 + len2; i2++) {
1206                                 Statement s2 = ss2.get(i2);
1207                                 int d = compareObject(s1.getObject(), s2.getObject());
1208                                 if (d == 0) {
1209                                         for (Resource t : strong) {
1210                                                  if (s1.getPredicate().equals(t) || g.isSubrelationOf(s1.getPredicate(), t)) {
1211                                                          d = 1;
1212                                                          break;
1213                                                  }
1214                                         }
1215                                 }
1216                                 diff.add(d);
1217                         }
1218                         differences.add(diff);
1219                 }
1220                 // difference, left
1221                 MapList<Integer, Integer> priorities = new MapList<Integer, Integer>();
1222                 for (int i = 0; i < differences.size(); i++) {
1223                         List<Integer> list = differences.get(i);
1224                         for (int j = 0; j < list.size(); j++) {
1225                                 priorities.add(list.get(j), i);
1226                         }
1227                 }
1228
1229                 Integer[] pris = priorities.getKeys(new Integer[]{});
1230                 Arrays.sort(pris);
1231                 boolean matchFail = priorityMatching(ss1, off1, len1, ss2, off2, len2, pris, differences, priorities, used1, used2, objectsLeft, objectsRight, false);
1232                 if (matchFail)
1233                         matchFail = priorityMatching(ss1, off1, len1, ss2, off2, len2, pris, differences, priorities, used1, used2, objectsLeft, objectsRight, objectsLeft == null);
1234                 if (unreliableLeft != null) {
1235                         if (matchFail) {
1236                                 // With match failure, statement matching was aborted. We must move all unmatched statements to unreliable.
1237                                 for (Integer pri : pris) {
1238                                         if (pri == 0 || pri == Integer.MAX_VALUE)
1239                                                 continue;
1240                                         priorityProcessUnmatched(ss1, off1, len1, ss2, off2, len2, differences, priorities, used1, used2, unreliableLeft, unreliableRight, pri);
1241                                 }
1242                         }
1243                         // Zero means unreliable comparison result, move all unmatched to unreliable.
1244                         if (org.simantics.utils.datastructures.Arrays.contains(pris, 0)) {
1245                                 priorityProcessUnmatched(ss1, off1, len1, ss2, off2, len2, differences, priorities, used1, used2, unreliableLeft, unreliableRight, 0);
1246                         }
1247                 }
1248 //      Previous version processed 0 priority statements, even when unreliableLeft collection was null.
1249 //              The problem was that property relations were not filtered in process() from "tested" statements.
1250 //              else {
1251 //                      if (org.simantics.utils.datastructures.Arrays.contains(pris, 0)) {
1252 //                              priorityProcessUnmatched(ss1, off1, len1, ss2, off2, len2, differences, priorities, used1, used2, unreliableLeft, unreliableRight, 0);
1253 //                      }
1254 //              }
1255                 // Report unmatched statements as changes.
1256                 for (int i1 = off1; i1 < off1 + len1; i1++) {
1257                         if (!used1[i1-off1]) {
1258                                 if (DEBUG) System.out.println("Compare Object deletion " + printStatement(g,ss1.get(i1)));
1259                                 addDeletion(ss1.get(i1));
1260                         }
1261                 }
1262                 for (int i2 = off2; i2 < off2 + len2; i2++) {
1263                         if (!used2[i2-off2]) {
1264                                 if (DEBUG) System.out.println("Compare Object addition " + printStatement(g,ss2.get(i2)));
1265                                 addAddition(ss2.get(i2));
1266                         }
1267                 }
1268         }
1269
1270         /**
1271          * Moves unmatched statements to unreliable collections.
1272          * @param ss1
1273          * @param off1
1274          * @param len1
1275          * @param ss2
1276          * @param off2
1277          * @param len2
1278          * @param differences
1279          * @param priorities
1280          * @param used1
1281          * @param used2
1282          * @param unreliableLeft
1283          * @param unreliableRight
1284          * @param pri
1285          */
1286         private void priorityProcessUnmatched(List<Statement> ss1, int off1, int len1, List<Statement> ss2, int off2, int len2,List<List<Integer>> differences,MapList<Integer, Integer> priorities,boolean used1[],boolean used2[], Collection<Statement> unreliableLeft, Collection<Statement> unreliableRight, int pri) {
1287                 Set<Statement> s1s = new HashSet<Statement>();
1288                 Set<Statement> s2s = new HashSet<Statement>();
1289                 Set<Integer> s1i = new HashSet<Integer>();
1290                 Set<Integer> s2i = new HashSet<Integer>();
1291                 List<Integer> i1s = priorities.getValues(pri);
1292                 for (Integer i1 : i1s) {
1293                         if (used1[i1])
1294                                 continue;
1295                         List<Integer> i2diff = differences.get(i1);
1296                         for (int i2 = 0; i2 < i2diff.size(); i2++) {
1297                                 if (i2diff.get(i2) == pri) {
1298                                         if (used2[i2])
1299                                                 continue;
1300                                         Statement s1  = ss1.get(i1+off1);
1301                                         Statement s2  = ss2.get(i2+off2);
1302                                         s1s.add(s1);
1303                                         s2s.add(s2);
1304                                         s1i.add(i1);
1305                                         s2i.add(i2);
1306                                 }
1307                         }
1308                 }
1309                 if (unreliableLeft != null) {
1310                         unreliableLeft.addAll(s1s);
1311                         unreliableRight.addAll(s2s);
1312                 }
1313                 for (Integer i : s1i)
1314                         used1[i] = true;
1315                 for (Integer i : s2i)
1316                         used2[i] = true;
1317         }
1318         /**
1319          * Matches left and right side statements.
1320          *
1321          * When there are two or more equally good matching objects, the behaviour depends on force flag.
1322          *   False:  Matching is aborted and matchFail is returned (method return true).
1323          *   True: equally good matches will be paired randomly. Method always returns false.
1324          * @param ss1
1325          * @param off1
1326          * @param len1
1327          * @param ss2
1328          * @param off2
1329          * @param len2
1330          * @param pris
1331          * @param differences
1332          * @param priorities
1333          * @param used1
1334          * @param used2
1335          * @param objectsLeft
1336          * @param objectsRight
1337          * @param force
1338          * @return
1339          * @throws DatabaseException
1340          */
1341         private boolean priorityMatching(List<Statement> ss1, int off1, int len1, List<Statement> ss2, int off2, int len2, Integer[] pris, List<List<Integer>> differences, MapList<Integer, Integer> priorities, boolean used1[],boolean used2[], Collection<Resource> objectsLeft, Collection<Resource> objectsRight, boolean force) throws DatabaseException {
1342                 boolean matchFail = false;
1343                 for (Integer pri : pris) {
1344                         if (pri == Integer.MAX_VALUE) {
1345
1346                         } else if (pri == 0) {
1347
1348                         } else {
1349                                 List<Integer> i1s = priorities.getValues(pri);
1350
1351                                 for (Integer i1 : i1s) {
1352                                         if (used1[i1])
1353                                                 continue;
1354                                         List<Integer> i2diff = differences.get(i1);
1355                                         List<Integer> matches = new ArrayList<Integer>();
1356                                         for (int i2 = 0; i2 < i2diff.size(); i2++) {
1357                                                 if (i2diff.get(i2) == pri) {
1358                                                         if (used2[i2])
1359                                                                 continue;
1360                                                         matches.add(i2);
1361                                                 }
1362                                         }
1363                                         if (matches.size() == 1 || (force && matches.size() > 1)) {
1364                                                 int i2 = matches.get(0);
1365                                                 used1[i1] = true;
1366                                                 used2[i2] = true;
1367                                                 Statement s1  = ss1.get(i1+off1);
1368                                                 Statement s2  = ss2.get(i2+off2);
1369
1370                                                 if (objectsLeft != null) {
1371                                                         objectsLeft.add(s1.getObject());
1372                                                         objectsRight.add(s2.getObject());
1373                                                 }
1374                                                 addComparable(s1, s2);
1375                                         } else if (matches.size() > 1) {
1376                                                 matchFail = true;
1377                                         }
1378                                 }
1379                                 if (matchFail)
1380                                         break;
1381                         }
1382                 }
1383                 return matchFail;
1384         }
1385
1386
1387         /**
1388          * compares properties, assumes functional relations
1389          * @param r1
1390          * @param r2
1391          * @throws DatabaseException
1392          */
1393         private void compareProps(Resource r1, Resource r2) throws DatabaseException {
1394                 if (DEBUG) System.out.println("compareProps " + r1 + " " + NameUtils.getSafeName(g, r1) + " " + r2 + " " + NameUtils.getSafeName(g, r2));
1395                 List<Statement> ss1 = new ArrayList<Statement>();
1396                 List<Statement> ss2 = new ArrayList<Statement>();
1397                 ss1.addAll(g.getStatements(r1, b.HasProperty));
1398                 ss2.addAll(g.getStatements(r2, b.HasProperty));
1399                 ss1 = filterNonTested(ss1);
1400                 ss2 = filterNonTested(ss2);
1401                 sortStatement(ss1, ss2);
1402                 // getStatements(r1, b.HasProperty) returns both instance and asserted statements for the same property relation.
1403                 ss1 = filterAssertedDuplicates(r1, ss1);
1404                 ss2 = filterAssertedDuplicates(r2, ss2);
1405
1406                 int i1 = 0;
1407                 int i2 = 0;
1408
1409                 while (true) {
1410                         if (i1 >= ss1.size()) {
1411                                 if (i2 >= ss2.size())
1412                                         break;
1413                                 else {
1414                                         while (i2 < ss2.size()) {
1415                                                 Statement s = ss2.get(i2);
1416                                                 if (DEBUG) System.out.println("Compare Prop diff2 " + printStatement(g,s));
1417                                                 if (!s.isAsserted(r2))
1418                                                         addAddition(s);
1419                                                 i2++;
1420                                         }
1421                                         break;
1422                                 }
1423                         } else if (i2 >= ss2.size()) {
1424                                 while (i1 < ss1.size()) {
1425                                         Statement s = ss1.get(i1);
1426                                         if (DEBUG) System.out.println("Compare Prop diff1 " + printStatement(g,s));
1427                                         if (!s.isAsserted(r1))
1428                                                 addDeletion(s);
1429                                         i1++;
1430                                 }
1431                                 break;
1432                         }
1433                         Statement s1 = ss1.get(i1);
1434                         Statement s2 = ss2.get(i2);
1435                         if (s1.isAsserted(r1) && s2.isAsserted(r2)) {
1436                                 i1++;
1437                                 i2++;
1438                                 continue;
1439                         }
1440                         int c = scomp.compare(s1, s2);
1441                         switch (c) {
1442                                 case 0:{
1443                                         boolean b1 = g.hasValue(s1.getObject());
1444                                         boolean b2 = g.hasValue(s2.getObject());
1445                                         boolean a1 = s1.isAsserted(r1);
1446                                         boolean a2 = s2.isAsserted(r2);
1447                                         if (b1 == b2) {
1448                                                 if (b1) {
1449                                                         // Literals
1450                                                         boolean eq = compareValue(g,b,s1.getObject(), s2.getObject());
1451                                                         if (!eq) {
1452                                                                 addModification(r1,s1,r2,s2);
1453                                                         }
1454                                                         if (!a1 && !a2)
1455                                                                 addComparable(s1, s2);
1456                                                 } else {
1457                                                         // Non literal properties.
1458                                                         int comp = comparator.compare(g, s1.getObject(), s2.getObject());
1459                                                         if (comp == ResourceComparator.NO_MATCH) {
1460                                                                 addModification(r1,s1,r2,s2);
1461                                                         } else if (comp != ResourceComparator.EXACT_MATCH) {
1462                                                                 if (!s1.getObject().equals(s1.getSubject()) && !s2.getObject().equals(s2.getSubject())) {
1463                                                                         if (!a1 && !a2) {
1464                                                                                 // compare props matches objects, so we can call that only for non asserted statements
1465                                                                                 compareProps(s1.getObject(), s2.getObject());
1466                                                                         } else if (a1 && a2) {
1467                                                                                 // TODO : compare asserted statements?
1468                                                                         } else {
1469                                                                                 addModification(r1,s1,r2,s2);
1470                                                                         }
1471                                                                 } else {
1472                                                                         addModification(r1,s1,r2,s2);
1473                                                                 }
1474                                                         } else {
1475                                                                 // Exact match, nothing to do.
1476                                                                 if (!a1 && !a2)
1477                                                                         addComparable(s1, s2);
1478                                                         }
1479                                                 }
1480                                         } else {
1481                                                 addModification(r1,s1,r2,s2);
1482                                         }
1483                                         i1++;
1484                                         i2++;
1485                                         break;
1486                                 }
1487                                 case -1:{
1488                                         if (DEBUG) System.out.println("Compare Prop diff1s " + printStatement(g,s1));
1489                                         // Use modification, since deletions do not support asserted statements
1490                                         addModification(r1,s1,r2,null);
1491                                         //addDeletion(s1);
1492                                         i1++;
1493                                         break;
1494                                 }
1495
1496                                 case 1:{
1497                                         if (DEBUG) System.out.println("Compare Prop diff2s " + printStatement(g,s2));
1498                                         // Use modification, since additions do not support asserted statements
1499                                         addModification(r1,null,r2,s2);
1500                                         //addAddition(s2);
1501                                         i2++;
1502                                         break;
1503                                 }
1504                         }
1505
1506                 }
1507
1508                 ss1.clear();
1509                 ss2.clear();
1510
1511         }
1512
1513         public static boolean compareValue(ReadGraph g, Layer0 b, Resource r1, Resource r2) throws DatabaseException {
1514                 Resource t1 = g.getSingleType(r1);
1515                 Resource t2 = g.getSingleType(r2);
1516                 if (!t1.equals(t2))
1517                         return false;
1518                 if (t1.equals(b.Integer)) {
1519                         int v1 = g.getValue(r1, Bindings.INTEGER);
1520                         int v2 = g.getValue(r2, Bindings.INTEGER);
1521                         return v1 == v2;
1522                 } else if (t1.equals(b.Float)) {
1523                         float v1 = g.getValue(r1, Bindings.FLOAT);
1524                         float v2 = g.getValue(r2, Bindings.FLOAT);
1525                         return v1 == v2;
1526                 } else if (t1.equals(b.Double)) {
1527                         double v1 = g.getValue(r1, Bindings.DOUBLE);
1528                         double v2 = g.getValue(r2, Bindings.DOUBLE);
1529                         return v1 == v2;
1530                 } else if (t1.equals(b.String)) {
1531                         String v1 = g.getValue(r1, Bindings.STRING);
1532                         String v2 = g.getValue(r2, Bindings.STRING);
1533                         return v1.equals(v2);
1534                 } else if (t1.equals(b.Boolean)) {
1535                         boolean v1 = g.getValue(r1, Bindings.BOOLEAN);
1536                         boolean v2 = g.getValue(r2, Bindings.BOOLEAN);
1537                         return v1 == v2;
1538                 } else if (t1.equals(b.Byte)) {
1539                         byte v1 = g.getValue(r1, Bindings.BYTE);
1540                         byte v2 = g.getValue(r2, Bindings.BYTE);
1541                         return v1 == v2;
1542                 } else if (t1.equals(b.Long)) {
1543                         long v1 = g.getValue(r1, Bindings.LONG);
1544                         long v2 = g.getValue(r2, Bindings.LONG);
1545                         return v1 == v2;
1546                 } else if (t1.equals(b.IntegerArray)) {
1547                         int[] v1 = g.getValue(r1, Bindings.INT_ARRAY);
1548                         int[] v2 = g.getValue(r2, Bindings.INT_ARRAY);
1549                         return Arrays.equals(v1,v2);
1550                 } else if (t1.equals(b.FloatArray)) {
1551                         float[] v1 = g.getValue(r1, Bindings.FLOAT_ARRAY);
1552                         float[] v2 = g.getValue(r2, Bindings.FLOAT_ARRAY);
1553                         return Arrays.equals(v1,v2);
1554                 } else if (t1.equals(b.DoubleArray)) {
1555                         double[] v1 = g.getValue(r1, Bindings.DOUBLE_ARRAY);
1556                         double[] v2 = g.getValue(r2, Bindings.DOUBLE_ARRAY);
1557                         return Arrays.equals(v1,v2);
1558                 } else if (t1.equals(b.StringArray)) {
1559                         String[] v1 = g.getValue(r1, Bindings.STRING_ARRAY);
1560                         String[] v2 = g.getValue(r2, Bindings.STRING_ARRAY);
1561                         return Arrays.equals(v1,v2);
1562                 } else if (t1.equals(b.BooleanArray)) {
1563                         boolean[] v1 = g.getValue(r1, Bindings.BOOLEAN_ARRAY);
1564                         boolean[] v2 = g.getValue(r2, Bindings.BOOLEAN_ARRAY);
1565                         return Arrays.equals(v1,v2);
1566                 } else if (t1.equals(b.ByteArray)) {
1567                         byte[] v1 = g.getValue(r1, Bindings.BYTE_ARRAY);
1568                         byte[] v2 = g.getValue(r2, Bindings.BYTE_ARRAY);
1569                         return Arrays.equals(v1,v2);
1570                 } else if (t1.equals(b.LongArray)) {
1571                         long[] v1 = g.getValue(r1, Bindings.LONG_ARRAY);
1572                         long[] v2 = g.getValue(r2, Bindings.LONG_ARRAY);
1573                         return Arrays.equals(v1,v2);
1574                 } else {
1575                         Object v1 = g.getValue(r1);
1576                         Object v2 = g.getValue(r2);
1577                         return compareValue(v1, v2);
1578                 }
1579
1580         }
1581
1582         public static boolean compareValue(Object v1, Object v2) {
1583                 if (v1 instanceof Object[] && v2 instanceof Object[])
1584                         return Arrays.deepEquals((Object[])v1, (Object[])v2);
1585                 else if (v1 instanceof int[] && v2 instanceof int[])
1586                         return Arrays.equals((int[])v1, (int[])v2);
1587                 else if (v1 instanceof float[] && v2 instanceof float[])
1588                         return Arrays.equals((float[])v1, (float[])v2);
1589                 else if (v1 instanceof double[] && v2 instanceof double[])
1590                         return Arrays.equals((double[])v1, (double[])v2);
1591                 else if (v1 instanceof long[] && v2 instanceof long[])
1592                         return  Arrays.equals((long[])v1, (long[])v2);
1593                 else if (v1 instanceof byte[] && v2 instanceof byte[])
1594                         return Arrays.equals((byte[])v1, (byte[])v2);
1595                 else if (v1 instanceof boolean[] && v2 instanceof boolean[])
1596                         return Arrays.equals((boolean[])v1, (boolean[])v2);
1597                 else
1598                         return v1.equals(v2);
1599         }
1600
1601
1602         public class PredicateComparator implements Comparator<Statement> {
1603                 @Override
1604                 public int compare(Statement o1, Statement o2) {
1605                         if (comparableResources.contains(o1.getPredicate(), o2.getPredicate()))
1606                                 return 0;
1607                         if (o1.getPredicate().getResourceId() < o2.getPredicate().getResourceId())
1608                                 return -1;
1609                         if (o1.getPredicate().getResourceId() > o2.getPredicate().getResourceId())
1610                                 return 1;
1611                         return 0;
1612                 }
1613         }
1614
1615         public class SubjectComparator implements Comparator<Statement> {
1616                 @Override
1617                 public int compare(Statement o1, Statement o2) {
1618                         if (comparableResources.contains(o1.getSubject(), o2.getSubject()))
1619                                 return 0;
1620                         if (o1.getSubject().getResourceId() < o2.getSubject().getResourceId())
1621                                 return -1;
1622                         if (o1.getSubject().getResourceId() > o2.getSubject().getResourceId())
1623                                 return 1;
1624                         return 0;
1625                 }
1626         }
1627
1628         public class ObjectComparator implements Comparator<Statement> {
1629                 @Override
1630                 public int compare(Statement o1, Statement o2) {
1631                         if (comparableResources.contains(o1.getObject(), o2.getObject()))
1632                                 return 0;
1633                         if (o1.getObject().getResourceId() < o2.getObject().getResourceId())
1634                                 return -1;
1635                         if (o1.getObject().getResourceId() > o2.getObject().getResourceId())
1636                                 return 1;
1637                         return 0;
1638                 }
1639         }
1640
1641         public static class FullStatementComparator implements Comparator<Statement> {
1642                 @Override
1643                 public int compare(Statement o1, Statement o2) {
1644                         if (o1.getSubject().getResourceId() < o2.getSubject().getResourceId())
1645                                 return -1;
1646                         if (o1.getSubject().getResourceId() > o2.getSubject().getResourceId())
1647                                 return 1;
1648                         if (o1.getPredicate().getResourceId() < o2.getPredicate().getResourceId())
1649                                 return -1;
1650                         if (o1.getPredicate().getResourceId() > o2.getPredicate().getResourceId())
1651                                 return 1;
1652                         if (o1.getObject().getResourceId() < o2.getObject().getResourceId())
1653                                 return -1;
1654                         if (o1.getObject().getResourceId() > o2.getObject().getResourceId())
1655                                 return 1;
1656                         return 0;
1657                 }
1658         }
1659
1660         public class ResComparator implements Comparator<Resource> {
1661                 @Override
1662                 public int compare(Resource o1, Resource o2) {
1663                         if (comparableResources.contains(o1, o2))
1664                                 return 0;
1665                         if (o1.getResourceId() < o2.getResourceId())
1666                                 return -1;
1667                         if (o1.getResourceId() > o2.getResourceId())
1668                                 return 1;
1669                         return 0;
1670                 }
1671         }
1672
1673 }