package org.simantics.db.tests.performance.read; import java.io.IOException; import java.io.UTFDataFormatException; import java.nio.charset.Charset; import java.util.Set; import java.util.Vector; import org.simantics.databoard.Bindings; import org.simantics.databoard.Datatypes; import org.simantics.databoard.binding.Binding; import org.simantics.databoard.binding.impl.StringBindingDefault; import org.simantics.databoard.serialization.SerializationException; import org.simantics.databoard.serialization.Serializer; import org.simantics.databoard.serialization.impl.ModifiedUTF8StringSerializer; import org.simantics.db.AsyncReadGraph; import org.simantics.db.ReadGraph; import org.simantics.db.RelationInfo; import org.simantics.db.Resource; import org.simantics.db.Session; import org.simantics.db.VirtualGraph; import org.simantics.db.WriteGraph; import org.simantics.db.WriteOnlyGraph; import org.simantics.db.common.request.ReadRequest; import org.simantics.db.common.request.WriteOnlyResultRequest; import org.simantics.db.common.request.WriteResultRequest; import org.simantics.db.exception.AssumptionException; import org.simantics.db.exception.DatabaseException; import org.simantics.db.procedure.AsyncMultiProcedure; import org.simantics.db.procedure.AsyncProcedure; import org.simantics.db.request.AsyncRead; import org.simantics.db.request.Read; import org.simantics.db.service.ClusterBuilder; import org.simantics.db.service.ClusterBuilder.ResourceHandle; import org.simantics.db.service.ClusterBuilder.StatementHandle; import org.simantics.db.service.DirectQuerySupport; import org.simantics.db.service.QueryControl; import org.simantics.db.service.SerialisationSupport; import org.simantics.db.tests.performance.read.HierarchicalNames.FastStringSerializer.FastStringSerializer2; import org.simantics.layer0.Layer0; import org.simantics.utils.datastructures.Pair; /* * The model contains: * -2M resources * -1M string literals * -6M statements */ public class HierarchicalNames { public static class FastStringSerializer extends ModifiedUTF8StringSerializer { public static final Charset UTF8 = Charset.forName("utf-8"); public static FastStringSerializer INSTANCE = new FastStringSerializer(); public FastStringSerializer() { super(FastStringBinding.INSTANCE); } static byte[] writeUTF(String str) throws IOException { int strlen = str.length(); int utflen = 0; int c, count = 0; /* use charAt instead of copying String to char array */ for (int i = 0; i < strlen; i++) { c = str.charAt(i); if ((c >= 0x0001) && (c <= 0x007F)) { utflen++; } else if (c > 0x07FF) { utflen += 3; } else { utflen += 2; } } if (utflen > 65535) throw new UTFDataFormatException( "encoded string too long: " + utflen + " bytes"); byte[] bytearr = new byte[utflen+2]; bytearr[count++] = (byte) ((utflen >>> 8) & 0xFF); bytearr[count++] = (byte) ((utflen >>> 0) & 0xFF); int i=0; for (i=0; i= 0x0001) && (c <= 0x007F))) break; bytearr[count++] = (byte) c; } for (;i < strlen; i++){ c = str.charAt(i); if ((c >= 0x0001) && (c <= 0x007F)) { bytearr[count++] = (byte) c; } else if (c > 0x07FF) { bytearr[count++] = (byte) (0xE0 | ((c >> 12) & 0x0F)); bytearr[count++] = (byte) (0x80 | ((c >> 6) & 0x3F)); bytearr[count++] = (byte) (0x80 | ((c >> 0) & 0x3F)); } else { bytearr[count++] = (byte) (0xC0 | ((c >> 6) & 0x1F)); bytearr[count++] = (byte) (0x80 | ((c >> 0) & 0x3F)); } } return bytearr; } public static class FastStringSerializer2 extends ModifiedUTF8StringSerializer { public static final Charset UTF8 = Charset.forName("utf-8"); public static FastStringSerializer2 INSTANCE = new FastStringSerializer2(); public FastStringSerializer2() { super(FastStringBinding.INSTANCE); } static byte[] writeUTF(String str) throws IOException { int strlen = str.length(); int utflen = 0; int c = 0; /* use charAt instead of copying String to char array */ for (int i = 0; i < strlen; i++) { c = str.charAt(i); if ((c >= 0x0001) && (c <= 0x007F)) { utflen++; } else if (c > 0x07FF) { utflen += 3; } else { utflen += 2; } } if (utflen > 65535) throw new UTFDataFormatException( "encoded string too long: " + utflen + " bytes"); int byteIndex = 0; byte[] bytearr; if(utflen < 0x80) { bytearr = new byte[utflen+1]; bytearr[byteIndex++] = ((byte)utflen); } else { utflen -= 0x80; if(utflen < 0x4000) { bytearr = new byte[utflen+2]; bytearr[byteIndex++] = (byte)( ((utflen&0x3f) | 0x80) ); bytearr[byteIndex++] = (byte)( (utflen>>>6) ); } else { utflen -= 0x4000; if(utflen < 0x200000) { bytearr = new byte[utflen+3]; bytearr[byteIndex++] = (byte)( ((utflen&0x1f) | 0xc0) ); bytearr[byteIndex++] = (byte)( ((utflen>>>5)&0xff) ); bytearr[byteIndex++] = (byte)( ((utflen>>>13)&0xff) ); } else { utflen -= 0x200000; if(utflen < 0x10000000) { bytearr = new byte[utflen+4]; bytearr[byteIndex++] = (byte)( ((utflen&0x0f) | 0xe0) ); bytearr[byteIndex++] = (byte)( ((utflen>>>4)&0xff) ); bytearr[byteIndex++] = (byte)( ((utflen>>>12)&0xff) ); bytearr[byteIndex++] = (byte)( ((utflen>>>20)&0xff) ); } else { utflen -= 0x10000000; bytearr = new byte[utflen+5]; bytearr[byteIndex++] = (byte)( ((utflen&0x07) | 0xf0) ); bytearr[byteIndex++] = (byte)( ((utflen>>>3)&0xff) ); bytearr[byteIndex++] = (byte)( ((utflen>>>11)&0xff) ); bytearr[byteIndex++] = (byte)( ((utflen>>>19)&0xff) ); bytearr[byteIndex++] = (byte)( ((utflen>>>27)&0xff) ); } } } } int i=0; for (i=0; i= 0x0001) && (c <= 0x007F))) break; bytearr[byteIndex++] = (byte)(c); } for (;i < strlen; i++){ c = str.charAt(i); if ((c >= 0x0001) && (c <= 0x007F)) { bytearr[byteIndex++] = (byte)( c ); } else if (c > 0x07FF) { bytearr[byteIndex++] = (byte)(0xE0 | ((c >> 12) & 0x0F)); bytearr[byteIndex++] = (byte)(0x80 | ((c >> 6) & 0x3F)); bytearr[byteIndex++] = (byte)(0x80 | ((c >> 0) & 0x3F)); } else { bytearr[byteIndex++] = (byte)(0xC0 | ((c >> 6) & 0x1F)); bytearr[byteIndex++] = (byte)(0x80 | ((c >> 0) & 0x3F)); } } return bytearr; } @Override public byte[] serialize(Object obj) throws IOException { try { return writeUTF((String)obj); } catch (IOException e) { throw new SerializationException(); } } } @Override public byte[] serialize(Object obj) throws IOException { try { return writeUTF((String)obj); } catch (IOException e) { throw new SerializationException(); } } // private static byte bytearr[] = new byte[80]; // private static char chararr[] = new char[80]; public final static String readUTF(byte[] bytearr) throws IOException { return "name"; // int utflen = (bytearr[0] << 8) + bytearr[1]; // // char[] chararr = new char[utflen*2]; // // int c, char2, char3; // int count = 2; // int chararr_count=0; // //// in.readFully(bytearr, 0, utflen); // // while (count < utflen + 2) { // c = (int) bytearr[count] & 0xff; // if (c > 127) break; // count++; // chararr[chararr_count++]=(char)c; // } // // while (count < utflen) { // c = (int) bytearr[count] & 0xff; // switch (c >> 4) { // case 0: case 1: case 2: case 3: case 4: case 5: case 6: case 7: // /* 0xxxxxxx*/ // count++; // chararr[chararr_count++]=(char)c; // break; // case 12: case 13: // /* 110x xxxx 10xx xxxx*/ // count += 2; // if (count > utflen) // throw new UTFDataFormatException( // "malformed input: partial character at end"); // char2 = (int) bytearr[count-1]; // if ((char2 & 0xC0) != 0x80) // throw new UTFDataFormatException( // "malformed input around byte " + count); // chararr[chararr_count++]=(char)(((c & 0x1F) << 6) | // (char2 & 0x3F)); // break; // case 14: // /* 1110 xxxx 10xx xxxx 10xx xxxx */ // count += 3; // if (count > utflen) // throw new UTFDataFormatException( // "malformed input: partial character at end"); // char2 = (int) bytearr[count-2]; // char3 = (int) bytearr[count-1]; // if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80)) // throw new UTFDataFormatException( // "malformed input around byte " + (count-1)); // chararr[chararr_count++]=(char)(((c & 0x0F) << 12) | // ((char2 & 0x3F) << 6) | // ((char3 & 0x3F) << 0)); // break; // default: // /* 10xx xxxx, 1111 xxxx */ // throw new UTFDataFormatException( // "malformed input around byte " + count); // } // } // if(chararr_count == 2*utflen) { // // The number of chars produced may be less than utflen // return new String(chararr); // } else { // // The number of chars produced may be less than utflen // return new String(chararr, 0, chararr_count); // } } @Override public Object deserialize(byte[] data) throws IOException { return readUTF(data); //return new String(data, UTF8); } } public static class FastStringBinding extends StringBindingDefault { public static FastStringBinding INSTANCE = new FastStringBinding(); public FastStringBinding() { super(Datatypes.STRING); } @Override public Serializer serializer() { return FastStringSerializer.INSTANCE; } } public static WriteOnlyResultRequest writeOnly(final String name, final int[] sizes) throws DatabaseException { return writeOnly(name, sizes, null); } public static WriteOnlyResultRequest writeOnly(final String name, final int[] sizes, VirtualGraph graph) throws DatabaseException { return new WriteOnlyResultRequest(graph) { @Override public Resource perform(WriteOnlyGraph graph) throws DatabaseException { //Binding binding = Bindings.STRING; // Serializer serializer = Bindings.STRING.serializer(); Serializer serializer = FastStringSerializer2.INSTANCE; Session session = graph.getService(Session.class); Layer0 b = Layer0.getInstance(session); graph.flushCluster(); ClusterBuilder builder = graph.getService(ClusterBuilder.class); SerialisationSupport ss = graph.getService(SerialisationSupport.class); ResourceHandle hasNameR = builder.resource(b.HasName); ResourceHandle nameOfR = builder.resource(b.NameOf); ResourceHandle consistsOfR = builder.resource(b.ConsistsOf); ResourceHandle partOfR = builder.resource(b.PartOf); ResourceHandle instanceOfR = builder.resource(b.InstanceOf); ResourceHandle libraryR = builder.resource(b.Library); ResourceHandle ontologyR = builder.resource(b.Ontology); ResourceHandle stringR = builder.resource(b.String); StatementHandle instanceOf = builder.newStatement(instanceOfR, libraryR); StatementHandle instanceOf2 = builder.newStatement(instanceOfR, ontologyR); StatementHandle instanceOfString = builder.newStatement(instanceOfR, stringR); ResourceHandle root = builder.newResource(); root.addStatement(instanceOf); ResourceHandle rootLiteral = builder.newResource(); rootLiteral.addStatement(instanceOfString); rootLiteral.addStatement(nameOfR, root); rootLiteral.addValue(name, serializer); root.addStatement(hasNameR, rootLiteral); // System.out.println("root: " + root.resource()); // System.out.println("literal: " + rootLiteral.resource()); // graph.addLiteral(root, b.HasName, b.NameOf, b.String, name, binding); // graph.claim(root, b.InstanceOf, null, b.Library); StatementHandle rootPart = builder.newStatement(partOfR, root); for(int i=0;i writeOnly2(final String name, final int[] sizes) throws DatabaseException { return writeOnly2(name, sizes, null); } public static WriteOnlyResultRequest writeOnly2(final String name, final int[] sizes, VirtualGraph graph) throws DatabaseException { return new WriteOnlyResultRequest(graph) { @Override public Resource perform(WriteOnlyGraph graph) throws DatabaseException { Binding binding = Bindings.STRING; Session session = graph.getService(Session.class); Layer0 b = Layer0.getInstance(session); Resource root = graph.newResource(); graph.addLiteral(root, b.HasName, b.NameOf, b.String, name, binding); graph.claim(root, b.InstanceOf, null, b.Library); for(int i=0;i write(final String name, final int[] sizes) throws DatabaseException { return write(name, sizes, null); } public static WriteResultRequest write(final String name, final int[] sizes, VirtualGraph graph) throws DatabaseException { return new WriteResultRequest(graph) { @Override public Resource perform(WriteGraph graph) throws DatabaseException { Binding binding = Bindings.STRING; Session session = graph.getService(Session.class); Layer0 b = Layer0.getInstance(session); Resource root = graph.newResource(); graph.addLiteral(root, b.HasName, b.NameOf, b.String, name, binding); graph.claim(root, b.InstanceOf, null, b.Library); for(int i=0;i write2(final String name, final int[] sizes) throws DatabaseException { return write(name, sizes, null); } public static WriteResultRequest write2(final String name, final int[] sizes, VirtualGraph graph) throws DatabaseException { return new WriteResultRequest(graph) { @Override public Resource perform(WriteGraph graph) throws DatabaseException { Binding binding = Bindings.STRING; Session session = graph.getService(Session.class); Layer0 b = Layer0.getInstance(session); Resource root = graph.newResource(); graph.claimLiteral(root, b.HasName, name, binding); graph.claim(root, b.InstanceOf, null, b.Library); for(int i=0;i readSync(final Resource resource) throws DatabaseException { return new ReadRequest() { private void readChildren(ReadGraph graph, Resource resource) throws DatabaseException { // System.err.println("child=" + resource); Layer0 L0 = Layer0.getInstance(graph); String name = graph.getPossibleRelatedValue(resource, L0.HasName, Bindings.STRING); // System.err.println("name=" + name); for(Resource child : graph.getObjects(resource, L0.ConsistsOf)) { readChildren(graph, child); } } @Override public void run(ReadGraph graph) throws DatabaseException { readChildren(graph, resource); } }; } public static Read readSync2(final Resource resource) throws DatabaseException { return new ReadRequest() { private void readChildren(ReadGraph graph, Resource resource) throws DatabaseException { Layer0 L0 = Layer0.getInstance(graph); for(Resource child : graph.getObjects(resource, L0.ConsistsOf)) { graph.getPossibleRelatedValue(child, L0.HasName); readChildren(graph, child); } } @Override public void run(ReadGraph graph) throws DatabaseException { readChildren(graph, resource); } }; } final private static boolean VALIDATE = false; public static Vector validation = new Vector(); public static Vector criteria = new Vector(); public static void validate() throws AssumptionException { if(VALIDATE) if(!validation.equals(criteria)) { for(String s : validation) System.err.println("-'" + s + "'"); throw new AssumptionException(""); } } public static Read readAsync(final Resource resource) { if(VALIDATE) { for(int i=0;i<244*64*64;i++) criteria.add("name"); validation.clear(); } class Process { final AsyncMultiProcedure structure; final AsyncProcedure names; Process(ReadGraph graph, Resource resource) { final Layer0 L0 = Layer0.getInstance(graph); final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class); final QueryControl control = graph.getService(QueryControl.class); names = dqs.compilePossibleRelatedValue(graph, L0.HasName, new AsyncProcedure() { @Override public void execute(AsyncReadGraph graph, String name) { if(VALIDATE) validation.add(name); // System.err.println("af=" + name); } @Override public void exception(AsyncReadGraph graph, Throwable throwable) { throwable.printStackTrace(); } }); structure = dqs.compileForEachObject(graph, L0.ConsistsOf, new AsyncMultiProcedure() { @Override public void execute(AsyncReadGraph graph, Resource child) { if(control.scheduleByCluster(graph, child, this)) { dqs.forEachObjectCompiled(graph, child, structure); dqs.forPossibleRelatedValueCompiled(graph, child, names); } } @Override public void finished(AsyncReadGraph graph) { } @Override public void exception(AsyncReadGraph graph, Throwable throwable) { throwable.printStackTrace(); } }); dqs.forEachObjectCompiled(graph, resource, structure); dqs.forPossibleRelatedValueCompiled(graph, resource, names); } } return new ReadRequest() { @Override public void run(ReadGraph graph) { new Process(graph, resource); } }; } public static void readAsyncLoop2(AsyncReadGraph graph, Layer0 L0, Resource resource, AsyncMultiProcedure procedure, AsyncProcedure procedure2) { graph.forEachObject(resource, L0.ConsistsOf, procedure); graph.forPossibleRelatedValue(resource, L0.HasName, FastStringBinding.INSTANCE, procedure2); } public static ReadRequest readAsync2(final Resource resource) { return new ReadRequest() { @Override public void run(ReadGraph graph) { final Layer0 L0 = Layer0.getInstance(graph); try { final AsyncProcedure names = new AsyncProcedure() { @Override public void execute(AsyncReadGraph graph, String name) { } @Override public void exception(AsyncReadGraph graph, Throwable throwable) { throwable.printStackTrace(); } }; readAsyncLoop2(graph, L0, resource, new AsyncMultiProcedure() { @Override public void execute(AsyncReadGraph graph, Resource child) { readAsyncLoop2(graph, L0, child, this, names); } @Override public void finished(AsyncReadGraph graph) { } @Override public void exception(AsyncReadGraph graph, Throwable throwable) { throwable.printStackTrace(); } }, names); } catch (Exception e) { e.printStackTrace(); } } }; } static int names = 0; public static void readAsyncTypesLoop(final DirectQuerySupport dqs, final Layer0 L0, AsyncReadGraph graph, final Resource resource, final RelationInfo consistsOf, final AsyncMultiProcedure procedure, final RelationInfo name, final Serializer serializer, final AsyncProcedure procedure2) { dqs.forPossibleType(graph, resource, new AsyncProcedure() { @Override public void execute(AsyncReadGraph graph, Resource type) { // System.err.println("affa"); graph.asyncRequest(new TypeSetAndString(type), new AsyncProcedure, String>>() { @Override public void execute(AsyncReadGraph graph, Pair, String> typeInfo) { Set types = typeInfo.first; if(types.contains(L0.Ontology)) { // dqs.forPossibleRelatedValue(graph, resource, name, serializer, new AsyncProcedure() { // // @Override // public void execute(AsyncReadGraph graph, String result) { //// if(names++ % 1000 == 0) System.err.println("names=" + names + "(" + result + ")"); // } // // @Override // public void exception(AsyncReadGraph graph, Throwable throwable) { // } // // }); } else if (types.contains(L0.Library)) { //dqs.forEachObject(graph, resource, consistsOf, procedure); } } @Override public void exception(AsyncReadGraph graph, Throwable throwable) { } }); } @Override public void exception(AsyncReadGraph graph, Throwable throwable) { } }); } public static Read readAsyncTypes(final Resource resource) { return new ReadRequest() { @Override public void run(ReadGraph graph) { final Layer0 L0 = Layer0.getInstance(graph); try { // graph.syncRequest(new TypeSetAndString(L0.Library), new TransientCacheListener, String>>()); // graph.syncRequest(new TypeSetAndString(L0.Ontology), new TransientCacheListener, String>>()); final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class); final RelationInfo consistsOf = graph.syncRequest(new AsyncRead() { @Override public void perform(AsyncReadGraph graph, AsyncProcedure procedure) { dqs.forRelationInfo(graph, L0.ConsistsOf, procedure); } @Override public int threadHash() { return hashCode(); } @Override public int getFlags() { return 0; } }); final RelationInfo name = graph.syncRequest(new AsyncRead() { @Override public void perform(AsyncReadGraph graph, AsyncProcedure procedure) { dqs.forRelationInfo(graph, L0.HasName, procedure); } @Override public int threadHash() { return hashCode(); } @Override public int getFlags() { return 0; } }); final Serializer serializer = FastStringSerializer.INSTANCE;//Bindings.getSerializerUnchecked( Bindings.STRING ); final AsyncProcedure names = new AsyncProcedure() { @Override public void execute(AsyncReadGraph graph, String name) { // System.out.println("exec: " + name); } @Override public void exception(AsyncReadGraph graph, Throwable throwable) { throwable.printStackTrace(); } }; // readAsyncLoop(dqs, graph, resource, new AsyncMultiProcedure() { // // @Override // public void execute(AsyncReadGraph graph, Resource child) { // readAsyncTypesLoop(dqs, L0, graph, child, consistsOf, this, name, serializer, names); // } // // @Override // public void finished(AsyncReadGraph graph) { // } // // @Override // public void exception(AsyncReadGraph graph, Throwable throwable) { // throwable.printStackTrace(); // } // // }, name, serializer, names); } catch (Exception e) { e.printStackTrace(); } } }; } }