bundles/org.simantics.graph.db/src/org/simantics/graph/db/TransferableGraphs.java

   1 /*******************************************************************************\r
   2  * Copyright (c) 2007, 2010 Association for Decentralized Information Management\r
   3  * in Industry THTH ry.\r
   4  * All rights reserved. This program and the accompanying materials\r
   5  * are made available under the terms of the Eclipse Public License v1.0\r
   6  * which accompanies this distribution, and is available at\r
   7  * http://www.eclipse.org/legal/epl-v10.html\r
   8  *\r
   9  * Contributors:\r
  10  *     VTT Technical Research Centre of Finland - initial API and implementation\r
  11  *******************************************************************************/\r
  12 package org.simantics.graph.db;\r
  13 \r
  14 import java.io.BufferedOutputStream;\r
  15 import java.io.DataInput;\r
  16 import java.io.DataOutput;\r
  17 import java.io.DataOutputStream;\r
  18 import java.io.File;\r
  19 import java.io.FileOutputStream;\r
  20 import java.io.IOException;\r
  21 import java.io.InputStream;\r
  22 import java.util.ArrayList;\r
  23 import java.util.Collection;\r
  24 import java.util.HashSet;\r
  25 import java.util.TreeMap;\r
  26 import java.util.UUID;\r
  27 \r
  28 import org.simantics.databoard.Accessors;\r
  29 import org.simantics.databoard.Bindings;\r
  30 import org.simantics.databoard.Datatypes;\r
  31 import org.simantics.databoard.accessor.error.AccessorConstructionException;\r
  32 import org.simantics.databoard.accessor.error.AccessorException;\r
  33 import org.simantics.databoard.accessor.file.FileVariantAccessor;\r
  34 import org.simantics.databoard.binding.Binding;\r
  35 import org.simantics.databoard.binding.error.BindingConstructionException;\r
  36 import org.simantics.databoard.binding.error.DatatypeConstructionException;\r
  37 import org.simantics.databoard.binding.mutable.Variant;\r
  38 import org.simantics.databoard.container.DataContainer;\r
  39 import org.simantics.databoard.container.DataContainers;\r
  40 import org.simantics.databoard.serialization.SerializationException;\r
  41 import org.simantics.databoard.serialization.Serializer;\r
  42 import org.simantics.databoard.type.Datatype;\r
  43 import org.simantics.db.ReadGraph;\r
  44 import org.simantics.db.RequestProcessor;\r
  45 import org.simantics.db.Resource;\r
  46 import org.simantics.db.Session;\r
  47 import org.simantics.db.VirtualGraph;\r
  48 import org.simantics.db.WriteGraph;\r
  49 import org.simantics.db.WriteOnlyGraph;\r
  50 import org.simantics.db.common.CommentMetadata;\r
  51 import org.simantics.db.common.request.ReadRequest;\r
  52 import org.simantics.db.common.request.WriteOnlyRequest;\r
  53 import org.simantics.db.common.request.WriteRequest;\r
  54 import org.simantics.db.exception.CancelTransactionException;\r
  55 import org.simantics.db.exception.DatabaseException;\r
  56 import org.simantics.db.service.SerialisationSupport;\r
  57 import org.simantics.db.service.VirtualGraphSupport;\r
  58 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceValueProcedure;\r
  59 import org.simantics.graph.diff.TransferableGraphDelta1;\r
  60 import org.simantics.graph.representation.Extensions;\r
  61 import org.simantics.graph.representation.External;\r
  62 import org.simantics.graph.representation.Identity;\r
  63 import org.simantics.graph.representation.TransferableGraph1;\r
  64 import org.simantics.graph.representation.Value;\r
  65 import org.simantics.utils.datastructures.BinaryFunction;\r
  66 \r
  67 import gnu.trove.list.array.TIntArrayList;\r
  68 import gnu.trove.map.hash.TObjectIntHashMap;\r
  69 \r
  70 public class TransferableGraphs {\r
  71 \r
  72         public static long[] importGraph(Session session, Object tg, IImportAdvisor advisor) throws DatabaseException, TransferableGraphException {\r
  73                 if (tg instanceof TransferableGraph1) \r
  74                 {\r
  75                         return importGraph1(session, (TransferableGraph1) tg, advisor);\r
  76                 }\r
  77                 throw new TransferableGraphException("Cannot import "+tg.getClass().getName());\r
  78         }\r
  79 \r
  80         public static long[] importGraph(WriteGraph g, Object tg, IImportAdvisor advisor) throws DatabaseException, TransferableGraphException {\r
  81                 if (tg instanceof TransferableGraph1) \r
  82                 {\r
  83                         return importGraph1(g, (TransferableGraph1) tg, advisor);\r
  84                 }\r
  85                 throw new TransferableGraphException("Cannot import "+tg.getClass().getName());\r
  86         }\r
  87         \r
  88         public static long[] importGraph(Session session, Object tg) throws DatabaseException, TransferableGraphException {\r
  89                 if (tg instanceof TransferableGraph1) \r
  90                 {\r
  91                         return importGraph1(session, (TransferableGraph1) tg);\r
  92                 }\r
  93                 throw new TransferableGraphException("Cannot import "+tg.getClass().getName());\r
  94         }\r
  95 \r
  96         public static long[] importGraph(WriteGraph g, Object tg) throws DatabaseException, TransferableGraphException {\r
  97                 if (tg instanceof TransferableGraph1) \r
  98                 {\r
  99                         return importGraph1(g, (TransferableGraph1) tg);\r
 100                 }\r
 101                 throw new TransferableGraphException("Cannot import "+tg.getClass().getName());\r
 102         }\r
 103         \r
 104         public static Collection<Resource> collectExternals(RequestProcessor processor, final TransferableGraph1 tg) throws DatabaseException, TransferableGraphException {\r
 105                 final TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg, new ImportAdvisor());\r
 106                 processor.syncRequest(new ReadRequest() {\r
 107                         @Override\r
 108                         public void run(ReadGraph graph) throws DatabaseException {\r
 109                                 process.prepare(graph);\r
 110                         }\r
 111                 });\r
 112                 HashSet<Resource> result = new HashSet<Resource>();\r
 113                 for(Identity id : tg.identities) {\r
 114                         if(id.definition instanceof External) {\r
 115                                 result.add(process.resources[id.resource]);\r
 116                         }\r
 117                 }\r
 118                 return result;\r
 119         }\r
 120         \r
 121         /**\r
 122          * Imports transferable graph version 1 to the database. Root advisor is used\r
 123          * to give identities to roots of the transferable graphs. It may be null,\r
 124          * in which case new resources are created for all roots but the root library.\r
 125          * \r
 126          * @param session\r
 127          * @param tg\r
 128      * @param advisor root advisor or <code>null</code>\r
 129          * @throws DatabaseException\r
 130          */\r
 131         public static long[] importGraph1(Session session, final TransferableGraph1 tg, final IImportAdvisor advisor_) throws DatabaseException, TransferableGraphException {\r
 132                 \r
 133                 final IImportAdvisor2 advisor = (advisor_ instanceof IImportAdvisor2) ? ((IImportAdvisor2)advisor_) : new WrapperAdvisor(advisor_);\r
 134                 \r
 135                 final TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg, \r
 136                                 advisor == null ? new ImportAdvisor() : advisor);\r
 137                 session.syncRequest(new ReadRequest() {\r
 138                         @Override\r
 139                         public void run(ReadGraph graph) throws DatabaseException {\r
 140                                 process.prepare(graph);\r
 141                         }\r
 142                 });             \r
 143                 session.syncRequest(new WriteOnlyRequest() {\r
 144                         @Override\r
 145                         public void perform(WriteOnlyGraph graph) throws DatabaseException {\r
 146                                 advisor.beforeWrite(graph, process);\r
 147                                 process.write(graph);\r
 148                                 advisor.afterWrite(graph, process);\r
 149                         }\r
 150                 });\r
 151                 return process.getResourceIds(\r
 152                                 session.getService(SerialisationSupport.class));\r
 153         }\r
 154 \r
 155         public static void importGraph1(Session session, final TransferableGraph1 tg, IImportAdvisor advisor, final BinaryFunction<Boolean, WriteOnlyGraph, TransferableGraphImportProcess> callback) throws DatabaseException, TransferableGraphException {\r
 156                 final TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg, \r
 157                                 advisor == null ? new ImportAdvisor() : advisor);\r
 158                 session.syncRequest(new ReadRequest() {\r
 159                         @Override\r
 160                         public void run(ReadGraph graph) throws DatabaseException {\r
 161                                 process.prepare(graph);\r
 162                         }\r
 163                 });\r
 164                 session.syncRequest(new WriteOnlyRequest() {\r
 165                         @Override\r
 166                         public void perform(WriteOnlyGraph graph) throws DatabaseException {\r
 167                                 process.write(graph);\r
 168                                 if(callback != null)\r
 169                                         callback.call(graph, process);\r
 170                         }\r
 171                 });\r
 172         }\r
 173 \r
 174     public static void importGraph1(Session session, final TransferableGraphSource tg, IImportAdvisor advisor) throws Exception {\r
 175         importGraph1(session, tg, advisor, null);\r
 176     }\r
 177 \r
 178     public static void importGraph1(Session session, final TransferableGraphSource tg, IImportAdvisor advisor, TGStatusMonitor monitor) throws DatabaseException {\r
 179         importGraph1(session, null, tg, advisor, monitor);\r
 180     }\r
 181 \r
 182     public static void importGraph1(Session session, VirtualGraph vg, final TransferableGraphSource tg, IImportAdvisor advisor_, TGStatusMonitor monitor) throws DatabaseException {\r
 183         \r
 184         final IImportAdvisor2 advisor = (advisor_ instanceof IImportAdvisor2) ? ((IImportAdvisor2)advisor_) : new WrapperAdvisor(advisor_);\r
 185 \r
 186                 final StreamingTransferableGraphImportProcess process = new StreamingTransferableGraphImportProcess(session, vg, tg, advisor);\r
 187                 session.syncRequest(new ReadRequest() {\r
 188                         @Override\r
 189                         public void run(ReadGraph graph) throws DatabaseException {\r
 190                                 try {\r
 191                                         process.prepare(graph);\r
 192                                 } catch (DatabaseException e) {\r
 193                                         throw e;\r
 194                                 } catch (Exception e) {\r
 195                                         throw new DatabaseException(e);\r
 196                                 }\r
 197                         }\r
 198                 });\r
 199                 session.syncRequest(new WriteOnlyRequest(vg) {\r
 200                         @Override\r
 201                         public void perform(WriteOnlyGraph graph) throws DatabaseException {\r
 202                                 try {\r
 203                                     advisor.beforeWrite(graph, process);\r
 204                                         process.write(graph);\r
 205                     advisor.afterWrite(graph, process);\r
 206                                 } catch (Exception e) {\r
 207                                         throw new DatabaseException(e);\r
 208                                 }\r
 209                         }\r
 210                 });\r
 211         }\r
 212 \r
 213         public static void importGraph1WithMonitor(Session session, final TransferableGraph1 tg, IImportAdvisor advisor_, TGStatusMonitor monitor) throws DatabaseException {\r
 214                 final IImportAdvisor2 advisor = (advisor_ instanceof IImportAdvisor2) ? ((IImportAdvisor2)advisor_) : new WrapperAdvisor(advisor_);\r
 215                 final TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg, \r
 216                                 advisor == null ? new ImportAdvisor() : advisor, monitor);\r
 217                 session.syncRequest(new ReadRequest() {\r
 218                         @Override\r
 219                         public void run(ReadGraph graph) throws DatabaseException {\r
 220                                 process.prepare(graph);\r
 221                         }\r
 222                 });\r
 223                 session.syncRequest(new WriteOnlyRequest() {\r
 224                         @Override\r
 225                         public void perform(WriteOnlyGraph graph) throws DatabaseException {\r
 226                                 advisor.beforeWrite(graph, process);\r
 227                                 process.write2(graph);\r
 228                                 advisor.afterWrite(graph, process);\r
 229                                 CommentMetadata comments = graph.getMetadata(CommentMetadata.class);\r
 230                                 comments.add("Imported transferable graph with " + tg.resourceCount + " resources");\r
 231                                 graph.addMetadata(comments);\r
 232                         }\r
 233                 });\r
 234         }\r
 235 \r
 236         public static void importGraph1WithChanges(Session session, final TransferableGraph1 tg, IImportAdvisor advisor, final BinaryFunction<Boolean, WriteGraph, TransferableGraphImportProcess> callback) throws DatabaseException, TransferableGraphException {\r
 237                 final TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg, \r
 238                                 advisor == null ? new ImportAdvisor() : advisor);\r
 239                 session.syncRequest(new ReadRequest() {\r
 240                         @Override\r
 241                         public void run(ReadGraph graph) throws DatabaseException {\r
 242                                 process.prepare(graph);\r
 243                         }\r
 244                 });\r
 245                 session.syncRequest(new WriteRequest() {\r
 246                         @Override\r
 247                         public void perform(WriteGraph graph) throws DatabaseException {\r
 248                                 process.write2(graph);\r
 249                 CommentMetadata comments = graph.getMetadata(CommentMetadata.class);\r
 250                 comments.add("Imported transferable graph with " + tg.resourceCount + " resources");\r
 251                 graph.addMetadata(comments);\r
 252                                 if(callback != null)\r
 253                                         callback.call(graph, process);\r
 254                         }\r
 255                 });\r
 256         }\r
 257         \r
 258         public static long[] importGraph1(Session session, final TransferableGraph1 tg) throws DatabaseException, TransferableGraphException {\r
 259                 final TransferableGraphImportProcess process = \r
 260                         new TransferableGraphImportProcess(tg, \r
 261                                 null);\r
 262                 session.syncRequest(new ReadRequest() {\r
 263                         @Override\r
 264                         public void run(ReadGraph graph) throws DatabaseException {\r
 265                                 process.prepare(graph);\r
 266                         }\r
 267                 });             \r
 268                 session.syncRequest(new WriteOnlyRequest() {\r
 269                         @Override\r
 270                         public void perform(WriteOnlyGraph graph) throws DatabaseException {\r
 271                                 process.write(graph);\r
 272                         }\r
 273                 });\r
 274                 return process.getResourceIds(\r
 275                                 session.getService(SerialisationSupport.class));\r
 276         }\r
 277         \r
 278         public static long[] importGraph1(WriteGraph graph, final TransferableGraph1 tg) throws DatabaseException, TransferableGraphException {\r
 279                 final TransferableGraphImportProcess process = \r
 280                         new TransferableGraphImportProcess(tg, \r
 281                                 null);\r
 282                 process.prepare(graph);\r
 283                 process.write2(graph);\r
 284                 return process.getResourceIds(\r
 285                                 graph.getSession().getService(SerialisationSupport.class));\r
 286         }\r
 287         \r
 288         /**\r
 289          * Import transferable graph version 1 to the database. Root advisor is used\r
 290          * to give identities to roots of the transferable graphs. It may be null,\r
 291          * in which case new resources are created for all roots but the root library.\r
 292          * \r
 293          * @param session\r
 294          * @param tg\r
 295      * @param advisor root advisor or <code>null</code>\r
 296          * @throws DatabaseException\r
 297          */\r
 298         public static long[] importGraph1(WriteGraph graph, TransferableGraph1 tg, IImportAdvisor advisor) throws DatabaseException {\r
 299             return importGraph1(graph, tg, advisor, null);\r
 300         }\r
 301 \r
 302         public static long[] importGraph1(WriteGraph graph, TransferableGraph1 tg, IImportAdvisor advisor, TGStatusMonitor monitor) throws DatabaseException {\r
 303                 TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg, \r
 304                                 advisor == null ? new ImportAdvisor() : advisor, monitor);\r
 305                 process.prepare(graph);\r
 306                 if(advisor instanceof IImportAdvisor2) ((IImportAdvisor2)advisor).beforeWrite(graph, process);\r
 307                 process.write2(graph);\r
 308                 if(advisor instanceof IImportAdvisor2) ((IImportAdvisor2)advisor).afterWrite(graph, process);\r
 309                 return process.getResourceIds(\r
 310                                 graph.getSession().getService(SerialisationSupport.class));\r
 311         }\r
 312 \r
 313         public static long[] applyDelta(WriteGraph graph, long[] oldResources, TransferableGraphDelta1 delta) throws DatabaseException {\r
 314                 SerialisationSupport serializer = \r
 315                         graph.getSession().getService(SerialisationSupport.class);\r
 316                 \r
 317                 TGToGraphMap aMap = new TGToGraphMap(delta.a);\r
 318                 aMap.addOldResources(serializer, oldResources);\r
 319                 aMap.deny(graph);\r
 320                 \r
 321                 TGToGraphMap bMap = new TGToGraphMap(delta.b);\r
 322                 bMap.addMappedOldResources(serializer, delta.aToB, aMap.getResources());\r
 323                 bMap.prepare(graph);\r
 324                 bMap.claim(graph);\r
 325                 \r
 326                 return bMap.getResources(serializer);\r
 327         }\r
 328         \r
 329         public static boolean hasChanges(ReadGraph graph, long[] oldResources, TransferableGraphDelta1 delta) throws DatabaseException {\r
 330                 \r
 331                 SerialisationSupport serializer = \r
 332                         graph.getSession().getService(SerialisationSupport.class);\r
 333                 \r
 334                 TGToGraphMap aMap = new TGToGraphMap(delta.a);\r
 335                 aMap.addOldResources(serializer, oldResources);\r
 336                 if(aMap.checkDeny(graph)) return true;\r
 337                 \r
 338                 TGToGraphMap bMap = new TGToGraphMap(delta.b);\r
 339                 bMap.addMappedOldResources(serializer, delta.aToB, aMap.getResources());\r
 340                 bMap.prepare(graph);\r
 341                 return bMap.checkClaim(graph);\r
 342                 \r
 343         }\r
 344 \r
 345         public static void uninstallGraph(WriteGraph writeGraph, TransferableGraph1 graph,\r
 346                         ImportAdvisor advisor) throws TransferableGraphException {\r
 347                 // TODO HANNU IMPLEMENTS\r
 348                 throw new UnsupportedOperationException();\r
 349         }       \r
 350         \r
 351         public static long[] importVirtualGraph(Session session, final VirtualGraph vg, final TransferableGraph1 tg, IImportAdvisor advisor) throws DatabaseException {\r
 352                 final TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg, \r
 353                                 advisor == null ? new ImportAdvisor() : advisor);\r
 354                 session.syncRequest(new ReadRequest() {\r
 355                         @Override\r
 356                         public void run(ReadGraph graph) throws DatabaseException {\r
 357                                 process.prepare(graph);\r
 358                         }\r
 359                 });\r
 360                 session.syncRequest(new WriteOnlyRequest(vg) {\r
 361                         @Override\r
 362                         public void perform(WriteOnlyGraph graph) throws DatabaseException {\r
 363                                 // Needed because process#write does not support virtual WriteOnlyGraph\r
 364                                 if (vg != null)\r
 365                                         process.write2(graph);\r
 366                                 else\r
 367                                         process.write(graph);\r
 368                         }\r
 369                         \r
 370                 });\r
 371                 return process.getResourceIds(session.getService(SerialisationSupport.class));\r
 372         }       \r
 373 \r
 374         public static long[] importVirtualGraph(WriteGraph graph, final TransferableGraph1 tg, IImportAdvisor advisor) throws DatabaseException {\r
 375                 final TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg, \r
 376                                 advisor == null ? new ImportAdvisor() : advisor);\r
 377                 process.prepare(graph);\r
 378                 process.write2(graph);\r
 379                 return process.getResourceIds(graph.getService(SerialisationSupport.class));\r
 380         }       \r
 381         \r
 382         public static TransferableGraph1 readGraph(File file) throws TransferableGraphException {\r
 383                 FileVariantAccessor va = null;\r
 384                 try {\r
 385                         va = Accessors.openAccessor(file);\r
 386                         Datatype type = va.getContentType();\r
 387                         if(type.equals(Datatypes.getDatatype(TransferableGraph1.class))) \r
 388                                 return  (TransferableGraph1)va.getContentValue(Bindings.getBinding(TransferableGraph1.class));\r
 389                         else\r
 390                                 throw new SerializationException("Unknown transferable graph data type.");\r
 391                 } catch (AccessorException e) {\r
 392                         throw new TransferableGraphException(e);\r
 393                 } catch (BindingConstructionException e) {\r
 394                         throw new TransferableGraphException(e);\r
 395                 } catch (SerializationException e) {\r
 396                         throw new TransferableGraphException(e);\r
 397                 } catch (AccessorConstructionException e) {\r
 398                         throw new TransferableGraphException(e);\r
 399                 } catch (DatatypeConstructionException e) {\r
 400                         throw new TransferableGraphException(e);\r
 401                 } finally {\r
 402                         if(va != null) {\r
 403                                 try {\r
 404                                         va.close();\r
 405                                 } catch (AccessorException e) {\r
 406                                 }\r
 407                         }\r
 408                 } \r
 409         }\r
 410         \r
 411         public static void importVirtualGraph(Session session, VirtualGraph vg, File file) throws TransferableGraphException {\r
 412                 try {\r
 413                         importVirtualGraph(session, vg, readGraph(file), new ImportAdvisor());\r
 414                 } catch (DatabaseException e) {\r
 415                         throw new TransferableGraphException(e);\r
 416                 }\r
 417         }\r
 418         \r
 419         public static VirtualGraph importVirtualGraph(Session session, File file) throws TransferableGraphException {\r
 420                 VirtualGraphSupport support = session.getService(VirtualGraphSupport.class);\r
 421                 VirtualGraph vg = support.getMemoryPersistent(UUID.randomUUID().toString());\r
 422                 importVirtualGraph(session, vg, file);\r
 423                 return vg;\r
 424         }\r
 425         \r
 426         public static void writeTransferableGraph(RequestProcessor processor, final String format, final TransferableGraphSource source, File target) throws Exception {\r
 427                 writeTransferableGraph(processor, format, 1, source, target);\r
 428         }\r
 429 \r
 430         public static void writeTransferableGraph(RequestProcessor processor, final String format, final int version, final TransferableGraphSource source, File target) throws Exception {\r
 431                 writeTransferableGraph(processor, format, version, new TreeMap<String,Variant>(), source, target);\r
 432         }\r
 433 \r
 434         public static void writeTransferableGraph(RequestProcessor processor, String format, int version, TreeMap<String, Variant> metadata, TransferableGraphSource source, File target) throws Exception {\r
 435                 writeTransferableGraph(processor, format, version, metadata, source, target, TGStatusMonitor.NULL_MONITOR);\r
 436         }\r
 437 \r
 438         public static void writeTransferableGraph(RequestProcessor processor, String format, int version, TreeMap<String, Variant> metadata, TransferableGraphSource source, File target, TGStatusMonitor monitor) throws Exception {\r
 439                 final Serializer datatypeSerializer = Bindings.getSerializerUnchecked(Datatype.class);\r
 440                 try (DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(target), 128*1024))) {\r
 441                         DataContainer container = new DataContainer(format, version, metadata, null);\r
 442                         DataContainers.writeHeader(out, container);\r
 443                         datatypeSerializer.serialize((DataOutput) out, Datatypes.getDatatypeUnchecked(TransferableGraph1.class));\r
 444                         writeTransferableGraph(processor, source, out, monitor);\r
 445                 }\r
 446         }\r
 447 \r
 448         private static TGStatusMonitor safeMonitor(TGStatusMonitor mon) {\r
 449                 return mon == null ? TGStatusMonitor.NULL_MONITOR : mon;\r
 450         }\r
 451 \r
 452         private static class CopyingInputStream extends InputStream {\r
 453                 public DataInput in;\r
 454                 public DataOutput out;\r
 455 \r
 456                 @Override\r
 457                 public int read() throws IOException {\r
 458                         int value = in.readUnsignedByte();\r
 459                         out.write(value);\r
 460                         return value;\r
 461                 }\r
 462         }\r
 463 \r
 464         private static long copy(byte[] buffer, DataInput in, DataOutput out, long bytesToCopy) throws IOException {\r
 465                 int read = 0;\r
 466                 long bufferLength = buffer.length;\r
 467                 while (read < bytesToCopy) {\r
 468                         int l = (int) Math.min(bufferLength, bytesToCopy-read);\r
 469                         in.readFully(buffer, 0, l);\r
 470                         out.write(buffer, 0, l);\r
 471                         read += l;\r
 472                 }\r
 473                 return read;\r
 474         }\r
 475 \r
 476         private static final int LITERAL_VALUE_IO_BUFFER_SIZE = 128 * 1024;\r
 477 \r
 478         private static void writeTransferableGraph(RequestProcessor processor, final TransferableGraphSource source, final DataOutput out, TGStatusMonitor monitor) throws Exception {\r
 479                 long start = System.nanoTime();\r
 480 \r
 481                 final Serializer datatypeSerializer = Bindings.getSerializerUnchecked(Datatype.class);\r
 482                 final Serializer identitySerializer = Bindings.getSerializerUnchecked(Identity.class);\r
 483                 final Serializer extensionSerializer = Bindings.getSerializerUnchecked(Extensions.class);\r
 484 \r
 485                 int totalCount = source.getIdentityCount() + source.getStatementCount()/4 + source.getValueCount();\r
 486                 TGStatusMonitor.Updater progress = new TGStatusMonitor.Updater(safeMonitor(monitor), totalCount);\r
 487 \r
 488                 out.writeInt(source.getResourceCount());\r
 489                 extensionSerializer.serialize(out, new Extensions(source.getExtensions()));\r
 490 \r
 491 //              System.err.println("resource count: " + source.getResourceCount());\r
 492 //              System.err.println("identity count: " + source.getIdentityCount());\r
 493 \r
 494                 byte[] buffer = new byte[LITERAL_VALUE_IO_BUFFER_SIZE];\r
 495 \r
 496                 processor.syncRequest(new ReadRequest() {\r
 497                         @Override\r
 498                         public void run(ReadGraph graph) throws DatabaseException {\r
 499                                 try {\r
 500                                         if (monitor.isCanceled())\r
 501                                                 throw new CancelTransactionException();\r
 502 \r
 503                                         out.writeInt(source.getIdentityCount());\r
 504                                         source.forIdentities(graph, value -> {\r
 505                                                 //System.err.println("id: " + value);\r
 506                                                 identitySerializer.serialize(out, value);\r
 507                                                 progress.worked(1);\r
 508                                         });\r
 509 \r
 510                                         if (monitor.isCanceled())\r
 511                                                 throw new CancelTransactionException();\r
 512 \r
 513                                         out.writeInt(source.getStatementCount());\r
 514                                         //System.err.println("stms: " + source.getStatementCount());\r
 515                                         //int[] counter = {0};\r
 516                                         source.forStatements(graph, r -> {\r
 517                                                 for (int i = 0; i < 4; ++i)\r
 518                                                         out.writeInt(r[i]);\r
 519                                                 //System.err.println("stm " + (counter[0]++) + ": " + r[0] + " " + r[1] + " " + r[2] + " " + r[3]);\r
 520                                                 progress.worked(1);\r
 521                                         });\r
 522 \r
 523                                         if (monitor.isCanceled())\r
 524                                                 throw new CancelTransactionException();\r
 525 \r
 526                                         out.writeInt(source.getValueCount());\r
 527                                         //System.err.println("vals: " + source.getValueCount());\r
 528                                         CopyingInputStream cis = new CopyingInputStream();\r
 529                                         cis.out = out;\r
 530                                         source.forValues2(graph, new TransferableGraphSourceValueProcedure() {\r
 531                                                 TObjectIntHashMap<Object> identities = new TObjectIntHashMap<>();\r
 532 \r
 533                                                 @Override\r
 534                                                 public void rawCopy(int resource, int length, DataInput input) throws Exception {\r
 535                                                         out.writeInt(resource);\r
 536                                                         long copied = copy(buffer, input, out, length);\r
 537                                                         assert copied == length;\r
 538                                                         //System.err.println("value " + (num++) + ": raw variant, " + length + " bytes, copied " + copied + " bytes");\r
 539                                                         progress.worked(1);\r
 540                                                 }\r
 541 \r
 542                                                 @Override\r
 543                                                 public void execute(int resource, Datatype type, DataInput input) throws Exception {\r
 544                                                         out.writeInt(resource);\r
 545                                                         identities.clear();\r
 546                                                         datatypeSerializer.serialize(out, identities, type);\r
 547                                                         Binding binding = Bindings.getBinding(type);\r
 548                                                         Serializer serializer = Bindings.getSerializer(binding);\r
 549                                                         cis.in = input;\r
 550                                                         serializer.skip(cis);\r
 551                                                         cis.in = null;\r
 552                                                         progress.worked(1);\r
 553                                                 }\r
 554                                         });\r
 555 \r
 556                                 } catch (DatabaseException e) {\r
 557                                         throw e;\r
 558                                 } catch (Exception e) {\r
 559                                         throw new DatabaseException(e);\r
 560                                 }\r
 561                         }\r
 562                 });\r
 563 \r
 564                 long end = System.nanoTime();\r
 565                 System.err.println("Wrote transferable graph in " + 1e-9*(end-start) + " seconds.");\r
 566         }\r
 567 \r
 568         public static TransferableGraph1 create(ReadGraph graph, TransferableGraphSource source) throws DatabaseException {\r
 569                 \r
 570                 final TIntArrayList statements = new TIntArrayList();\r
 571                 final ArrayList<Value> values = new ArrayList<>();\r
 572                 final ArrayList<Identity> identities = new ArrayList<>();\r
 573                 \r
 574                 try {\r
 575 \r
 576                         source.forStatements(graph, r -> statements.addAll(r));\r
 577                         source.forValues(graph, v -> values.add(v));\r
 578                         source.forIdentities(graph, i -> identities.add(i));\r
 579 \r
 580                         return new TransferableGraph1(source.getResourceCount(), \r
 581                                         identities.toArray(new Identity[identities.size()]),\r
 582                                         statements.toArray(),\r
 583                                         values.toArray(new Value[values.size()]),\r
 584                                         source.getExtensions());\r
 585                         \r
 586                 } catch (Exception e) {\r
 587                         \r
 588                         throw new DatabaseException(e);\r
 589                         \r
 590                 }\r
 591                 \r
 592         }\r
 593 \r
 594 }\r