X-Git-Url: https://gerrit.simantics.org/r/gitweb?a=blobdiff_plain;f=tests%2Forg.simantics.db.tests%2Fsrc%2Forg%2Fsimantics%2Fdb%2Ftests%2Fperformance%2Fread%2FHierarchicalNames.java;fp=tests%2Forg.simantics.db.tests%2Fsrc%2Forg%2Fsimantics%2Fdb%2Ftests%2Fperformance%2Fread%2FHierarchicalNames.java;h=18d6ce537940607764bf3b291b2606d323feb64a;hb=67fd62f9c742337ec80eef658192db198a0efaac;hp=0000000000000000000000000000000000000000;hpb=cde82ba81327d5515fdca362f7f4c70f5103ae80;p=simantics%2Fplatform.git diff --git a/tests/org.simantics.db.tests/src/org/simantics/db/tests/performance/read/HierarchicalNames.java b/tests/org.simantics.db.tests/src/org/simantics/db/tests/performance/read/HierarchicalNames.java new file mode 100644 index 000000000..18d6ce537 --- /dev/null +++ b/tests/org.simantics.db.tests/src/org/simantics/db/tests/performance/read/HierarchicalNames.java @@ -0,0 +1,978 @@ +package org.simantics.db.tests.performance.read; + +import java.io.IOException; +import java.io.UTFDataFormatException; +import java.nio.charset.Charset; +import java.util.Set; +import java.util.Vector; + +import org.simantics.databoard.Bindings; +import org.simantics.databoard.Datatypes; +import org.simantics.databoard.binding.Binding; +import org.simantics.databoard.binding.impl.StringBindingDefault; +import org.simantics.databoard.serialization.SerializationException; +import org.simantics.databoard.serialization.Serializer; +import org.simantics.databoard.serialization.impl.ModifiedUTF8StringSerializer; +import org.simantics.db.AsyncReadGraph; +import org.simantics.db.ReadGraph; +import org.simantics.db.RelationInfo; +import org.simantics.db.Resource; +import org.simantics.db.Session; +import org.simantics.db.VirtualGraph; +import org.simantics.db.WriteGraph; +import org.simantics.db.WriteOnlyGraph; +import org.simantics.db.common.request.ReadRequest; +import org.simantics.db.common.request.WriteOnlyResultRequest; +import org.simantics.db.common.request.WriteResultRequest; +import org.simantics.db.exception.AssumptionException; +import org.simantics.db.exception.DatabaseException; +import org.simantics.db.procedure.AsyncMultiProcedure; +import org.simantics.db.procedure.AsyncProcedure; +import org.simantics.db.request.AsyncRead; +import org.simantics.db.request.Read; +import org.simantics.db.service.ClusterBuilder; +import org.simantics.db.service.ClusterBuilder.ResourceHandle; +import org.simantics.db.service.ClusterBuilder.StatementHandle; +import org.simantics.db.service.DirectQuerySupport; +import org.simantics.db.service.QueryControl; +import org.simantics.db.service.SerialisationSupport; +import org.simantics.db.tests.performance.read.HierarchicalNames.FastStringSerializer.FastStringSerializer2; +import org.simantics.layer0.Layer0; +import org.simantics.utils.datastructures.Pair; + +/* + * The model contains: + * -2M resources + * -1M string literals + * -6M statements + */ +public class HierarchicalNames { + + public static class FastStringSerializer extends ModifiedUTF8StringSerializer { + + public static final Charset UTF8 = Charset.forName("utf-8"); + + public static FastStringSerializer INSTANCE = new FastStringSerializer(); + + public FastStringSerializer() { + super(FastStringBinding.INSTANCE); + } + + static byte[] writeUTF(String str) throws IOException { + + int strlen = str.length(); + int utflen = 0; + int c, count = 0; + + /* use charAt instead of copying String to char array */ + for (int i = 0; i < strlen; i++) { + c = str.charAt(i); + if ((c >= 0x0001) && (c <= 0x007F)) { + utflen++; + } else if (c > 0x07FF) { + utflen += 3; + } else { + utflen += 2; + } + } + + if (utflen > 65535) + throw new UTFDataFormatException( + "encoded string too long: " + utflen + " bytes"); + + byte[] bytearr = new byte[utflen+2]; + + bytearr[count++] = (byte) ((utflen >>> 8) & 0xFF); + bytearr[count++] = (byte) ((utflen >>> 0) & 0xFF); + + int i=0; + for (i=0; i= 0x0001) && (c <= 0x007F))) break; + bytearr[count++] = (byte) c; + } + + for (;i < strlen; i++){ + c = str.charAt(i); + if ((c >= 0x0001) && (c <= 0x007F)) { + bytearr[count++] = (byte) c; + + } else if (c > 0x07FF) { + bytearr[count++] = (byte) (0xE0 | ((c >> 12) & 0x0F)); + bytearr[count++] = (byte) (0x80 | ((c >> 6) & 0x3F)); + bytearr[count++] = (byte) (0x80 | ((c >> 0) & 0x3F)); + } else { + bytearr[count++] = (byte) (0xC0 | ((c >> 6) & 0x1F)); + bytearr[count++] = (byte) (0x80 | ((c >> 0) & 0x3F)); + } + } + return bytearr; + + } + + public static class FastStringSerializer2 extends ModifiedUTF8StringSerializer { + + public static final Charset UTF8 = Charset.forName("utf-8"); + + public static FastStringSerializer2 INSTANCE = new FastStringSerializer2(); + + public FastStringSerializer2() { + super(FastStringBinding.INSTANCE); + } + + static byte[] writeUTF(String str) throws IOException { + + int strlen = str.length(); + int utflen = 0; + int c = 0; + + /* use charAt instead of copying String to char array */ + for (int i = 0; i < strlen; i++) { + c = str.charAt(i); + if ((c >= 0x0001) && (c <= 0x007F)) { + utflen++; + } else if (c > 0x07FF) { + utflen += 3; + } else { + utflen += 2; + } + } + + if (utflen > 65535) + throw new UTFDataFormatException( + "encoded string too long: " + utflen + " bytes"); + + int byteIndex = 0; + byte[] bytearr; + + if(utflen < 0x80) { + bytearr = new byte[utflen+1]; + bytearr[byteIndex++] = ((byte)utflen); + } + else { + utflen -= 0x80; + if(utflen < 0x4000) { + bytearr = new byte[utflen+2]; + bytearr[byteIndex++] = (byte)( ((utflen&0x3f) | 0x80) ); + bytearr[byteIndex++] = (byte)( (utflen>>>6) ); + } + else { + utflen -= 0x4000; + if(utflen < 0x200000) { + bytearr = new byte[utflen+3]; + bytearr[byteIndex++] = (byte)( ((utflen&0x1f) | 0xc0) ); + bytearr[byteIndex++] = (byte)( ((utflen>>>5)&0xff) ); + bytearr[byteIndex++] = (byte)( ((utflen>>>13)&0xff) ); + } + else { + utflen -= 0x200000; + if(utflen < 0x10000000) { + bytearr = new byte[utflen+4]; + bytearr[byteIndex++] = (byte)( ((utflen&0x0f) | 0xe0) ); + bytearr[byteIndex++] = (byte)( ((utflen>>>4)&0xff) ); + bytearr[byteIndex++] = (byte)( ((utflen>>>12)&0xff) ); + bytearr[byteIndex++] = (byte)( ((utflen>>>20)&0xff) ); + } + else { + utflen -= 0x10000000; + bytearr = new byte[utflen+5]; + bytearr[byteIndex++] = (byte)( ((utflen&0x07) | 0xf0) ); + bytearr[byteIndex++] = (byte)( ((utflen>>>3)&0xff) ); + bytearr[byteIndex++] = (byte)( ((utflen>>>11)&0xff) ); + bytearr[byteIndex++] = (byte)( ((utflen>>>19)&0xff) ); + bytearr[byteIndex++] = (byte)( ((utflen>>>27)&0xff) ); + } + } + } + } + + + int i=0; + for (i=0; i= 0x0001) && (c <= 0x007F))) break; + bytearr[byteIndex++] = (byte)(c); + } + + for (;i < strlen; i++){ + c = str.charAt(i); + if ((c >= 0x0001) && (c <= 0x007F)) { + bytearr[byteIndex++] = (byte)( c ); + } else if (c > 0x07FF) { + bytearr[byteIndex++] = (byte)(0xE0 | ((c >> 12) & 0x0F)); + bytearr[byteIndex++] = (byte)(0x80 | ((c >> 6) & 0x3F)); + bytearr[byteIndex++] = (byte)(0x80 | ((c >> 0) & 0x3F)); + } else { + bytearr[byteIndex++] = (byte)(0xC0 | ((c >> 6) & 0x1F)); + bytearr[byteIndex++] = (byte)(0x80 | ((c >> 0) & 0x3F)); + } + } + + return bytearr; + + } + + @Override + public byte[] serialize(Object obj) throws IOException { + try { + return writeUTF((String)obj); + } catch (IOException e) { + throw new SerializationException(); + } + + } + + } + + + @Override + public byte[] serialize(Object obj) throws IOException { + try { + return writeUTF((String)obj); + } catch (IOException e) { + throw new SerializationException(); + } + + } + +// private static byte bytearr[] = new byte[80]; +// private static char chararr[] = new char[80]; + + public final static String readUTF(byte[] bytearr) throws IOException { + + return "name"; + +// int utflen = (bytearr[0] << 8) + bytearr[1]; +// +// char[] chararr = new char[utflen*2]; +// +// int c, char2, char3; +// int count = 2; +// int chararr_count=0; +// +//// in.readFully(bytearr, 0, utflen); +// +// while (count < utflen + 2) { +// c = (int) bytearr[count] & 0xff; +// if (c > 127) break; +// count++; +// chararr[chararr_count++]=(char)c; +// } +// +// while (count < utflen) { +// c = (int) bytearr[count] & 0xff; +// switch (c >> 4) { +// case 0: case 1: case 2: case 3: case 4: case 5: case 6: case 7: +// /* 0xxxxxxx*/ +// count++; +// chararr[chararr_count++]=(char)c; +// break; +// case 12: case 13: +// /* 110x xxxx 10xx xxxx*/ +// count += 2; +// if (count > utflen) +// throw new UTFDataFormatException( +// "malformed input: partial character at end"); +// char2 = (int) bytearr[count-1]; +// if ((char2 & 0xC0) != 0x80) +// throw new UTFDataFormatException( +// "malformed input around byte " + count); +// chararr[chararr_count++]=(char)(((c & 0x1F) << 6) | +// (char2 & 0x3F)); +// break; +// case 14: +// /* 1110 xxxx 10xx xxxx 10xx xxxx */ +// count += 3; +// if (count > utflen) +// throw new UTFDataFormatException( +// "malformed input: partial character at end"); +// char2 = (int) bytearr[count-2]; +// char3 = (int) bytearr[count-1]; +// if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80)) +// throw new UTFDataFormatException( +// "malformed input around byte " + (count-1)); +// chararr[chararr_count++]=(char)(((c & 0x0F) << 12) | +// ((char2 & 0x3F) << 6) | +// ((char3 & 0x3F) << 0)); +// break; +// default: +// /* 10xx xxxx, 1111 xxxx */ +// throw new UTFDataFormatException( +// "malformed input around byte " + count); +// } +// } +// if(chararr_count == 2*utflen) { +// // The number of chars produced may be less than utflen +// return new String(chararr); +// } else { +// // The number of chars produced may be less than utflen +// return new String(chararr, 0, chararr_count); +// } + + } + + @Override + public Object deserialize(byte[] data) throws IOException { + return readUTF(data); + //return new String(data, UTF8); + } + + } + + public static class FastStringBinding extends StringBindingDefault { + + public static FastStringBinding INSTANCE = new FastStringBinding(); + + public FastStringBinding() { + super(Datatypes.STRING); + } + + @Override + public Serializer serializer() { + return FastStringSerializer.INSTANCE; + } + + } + + public static WriteOnlyResultRequest writeOnly(final String name, final int[] sizes) throws DatabaseException { + return writeOnly(name, sizes, null); + } + + public static WriteOnlyResultRequest writeOnly(final String name, final int[] sizes, VirtualGraph graph) throws DatabaseException { + + return new WriteOnlyResultRequest(graph) { + + @Override + public Resource perform(WriteOnlyGraph graph) throws DatabaseException { + + //Binding binding = Bindings.STRING; +// Serializer serializer = Bindings.STRING.serializer(); + Serializer serializer = FastStringSerializer2.INSTANCE; + Session session = graph.getService(Session.class); + Layer0 b = Layer0.getInstance(session); + + graph.flushCluster(); + + ClusterBuilder builder = graph.getService(ClusterBuilder.class); + SerialisationSupport ss = graph.getService(SerialisationSupport.class); + + ResourceHandle hasNameR = builder.resource(b.HasName); + ResourceHandle nameOfR = builder.resource(b.NameOf); + ResourceHandle consistsOfR = builder.resource(b.ConsistsOf); + ResourceHandle partOfR = builder.resource(b.PartOf); + ResourceHandle instanceOfR = builder.resource(b.InstanceOf); + ResourceHandle libraryR = builder.resource(b.Library); + ResourceHandle ontologyR = builder.resource(b.Ontology); + ResourceHandle stringR = builder.resource(b.String); + + StatementHandle instanceOf = builder.newStatement(instanceOfR, libraryR); + StatementHandle instanceOf2 = builder.newStatement(instanceOfR, ontologyR); + StatementHandle instanceOfString = builder.newStatement(instanceOfR, stringR); + + ResourceHandle root = builder.newResource(); + root.addStatement(instanceOf); + + ResourceHandle rootLiteral = builder.newResource(); + rootLiteral.addStatement(instanceOfString); + rootLiteral.addStatement(nameOfR, root); + rootLiteral.addValue(name, serializer); + root.addStatement(hasNameR, rootLiteral); + +// System.out.println("root: " + root.resource()); +// System.out.println("literal: " + rootLiteral.resource()); + + // graph.addLiteral(root, b.HasName, b.NameOf, b.String, name, binding); + // graph.claim(root, b.InstanceOf, null, b.Library); + + StatementHandle rootPart = builder.newStatement(partOfR, root); + + for(int i=0;i writeOnly2(final String name, final int[] sizes) throws DatabaseException { + return writeOnly2(name, sizes, null); + } + + public static WriteOnlyResultRequest writeOnly2(final String name, final int[] sizes, VirtualGraph graph) throws DatabaseException { + + return new WriteOnlyResultRequest(graph) { + + @Override + public Resource perform(WriteOnlyGraph graph) throws DatabaseException { + + Binding binding = Bindings.STRING; + + Session session = graph.getService(Session.class); + Layer0 b = Layer0.getInstance(session); + + Resource root = graph.newResource(); + graph.addLiteral(root, b.HasName, b.NameOf, b.String, name, binding); + graph.claim(root, b.InstanceOf, null, b.Library); + + for(int i=0;i write(final String name, final int[] sizes) throws DatabaseException { + return write(name, sizes, null); + } + + public static WriteResultRequest write(final String name, final int[] sizes, VirtualGraph graph) throws DatabaseException { + + return new WriteResultRequest(graph) { + + @Override + public Resource perform(WriteGraph graph) throws DatabaseException { + + Binding binding = Bindings.STRING; + + Session session = graph.getService(Session.class); + Layer0 b = Layer0.getInstance(session); + + Resource root = graph.newResource(); + graph.addLiteral(root, b.HasName, b.NameOf, b.String, name, binding); + graph.claim(root, b.InstanceOf, null, b.Library); + + for(int i=0;i write2(final String name, final int[] sizes) throws DatabaseException { + return write(name, sizes, null); + } + + public static WriteResultRequest write2(final String name, final int[] sizes, VirtualGraph graph) throws DatabaseException { + + return new WriteResultRequest(graph) { + + @Override + public Resource perform(WriteGraph graph) throws DatabaseException { + + Binding binding = Bindings.STRING; + + Session session = graph.getService(Session.class); + Layer0 b = Layer0.getInstance(session); + + Resource root = graph.newResource(); + graph.claimLiteral(root, b.HasName, name, binding); + graph.claim(root, b.InstanceOf, null, b.Library); + + for(int i=0;i readSync(final Resource resource) throws DatabaseException { + + return new ReadRequest() { + + private void readChildren(ReadGraph graph, Resource resource) throws DatabaseException { +// System.err.println("child=" + resource); + Layer0 L0 = Layer0.getInstance(graph); + String name = graph.getPossibleRelatedValue(resource, L0.HasName, Bindings.STRING); +// System.err.println("name=" + name); + for(Resource child : graph.getObjects(resource, L0.ConsistsOf)) { + readChildren(graph, child); + } + } + + @Override + public void run(ReadGraph graph) throws DatabaseException { + readChildren(graph, resource); + } + + }; + + } + + public static Read readSync2(final Resource resource) throws DatabaseException { + + return new ReadRequest() { + + private void readChildren(ReadGraph graph, Resource resource) throws DatabaseException { + Layer0 L0 = Layer0.getInstance(graph); + for(Resource child : graph.getObjects(resource, L0.ConsistsOf)) { + graph.getPossibleRelatedValue(child, L0.HasName); + readChildren(graph, child); + } + } + + @Override + public void run(ReadGraph graph) throws DatabaseException { + readChildren(graph, resource); + } + + }; + + } + + + final private static boolean VALIDATE = false; + + public static Vector validation = new Vector(); + public static Vector criteria = new Vector(); + + public static void validate() throws AssumptionException { + if(VALIDATE) if(!validation.equals(criteria)) { + for(String s : validation) System.err.println("-'" + s + "'"); + throw new AssumptionException(""); + } + } + + public static Read readAsync(final Resource resource) { + + if(VALIDATE) { + for(int i=0;i<244*64*64;i++) criteria.add("name"); + validation.clear(); + } + + class Process { + + final AsyncMultiProcedure structure; + final AsyncProcedure names; + + Process(ReadGraph graph, Resource resource) { + + final Layer0 L0 = Layer0.getInstance(graph); + final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class); + final QueryControl control = graph.getService(QueryControl.class); + + names = dqs.compilePossibleRelatedValue(graph, L0.HasName, new AsyncProcedure() { + + @Override + public void execute(AsyncReadGraph graph, String name) { + if(VALIDATE) validation.add(name); +// System.err.println("af=" + name); + } + + @Override + public void exception(AsyncReadGraph graph, Throwable throwable) { + throwable.printStackTrace(); + } + + }); + + structure = dqs.compileForEachObject(graph, L0.ConsistsOf, new AsyncMultiProcedure() { + + @Override + public void execute(AsyncReadGraph graph, Resource child) { + if(control.scheduleByCluster(graph, child, this)) { + dqs.forEachObjectCompiled(graph, child, structure); + dqs.forPossibleRelatedValueCompiled(graph, child, names); + } + } + + @Override + public void finished(AsyncReadGraph graph) { + } + + @Override + public void exception(AsyncReadGraph graph, Throwable throwable) { + throwable.printStackTrace(); + } + + }); + + dqs.forEachObjectCompiled(graph, resource, structure); + dqs.forPossibleRelatedValueCompiled(graph, resource, names); + + } + + } + + return new ReadRequest() { + + @Override + public void run(ReadGraph graph) { + + new Process(graph, resource); + + } + + }; + + } + + public static void readAsyncLoop2(AsyncReadGraph graph, Layer0 L0, Resource resource, AsyncMultiProcedure procedure, AsyncProcedure procedure2) { + graph.forEachObject(resource, L0.ConsistsOf, procedure); + graph.forPossibleRelatedValue(resource, L0.HasName, FastStringBinding.INSTANCE, procedure2); + } + + public static ReadRequest readAsync2(final Resource resource) { + + return new ReadRequest() { + + @Override + public void run(ReadGraph graph) { + + final Layer0 L0 = Layer0.getInstance(graph); + + try { + + final AsyncProcedure names = new AsyncProcedure() { + + @Override + public void execute(AsyncReadGraph graph, String name) { + } + + @Override + public void exception(AsyncReadGraph graph, Throwable throwable) { + throwable.printStackTrace(); + } + + }; + + readAsyncLoop2(graph, L0, resource, new AsyncMultiProcedure() { + + @Override + public void execute(AsyncReadGraph graph, Resource child) { + readAsyncLoop2(graph, L0, child, this, names); + } + + @Override + public void finished(AsyncReadGraph graph) { + } + + @Override + public void exception(AsyncReadGraph graph, Throwable throwable) { + throwable.printStackTrace(); + } + + }, names); + + } catch (Exception e) { + + e.printStackTrace(); + + } + + } + + }; + + } + + static int names = 0; + + public static void readAsyncTypesLoop(final DirectQuerySupport dqs, final Layer0 L0, AsyncReadGraph graph, final Resource resource, final RelationInfo consistsOf, final AsyncMultiProcedure procedure, final RelationInfo name, final Serializer serializer, final AsyncProcedure procedure2) { + + dqs.forPossibleType(graph, resource, new AsyncProcedure() { + + @Override + public void execute(AsyncReadGraph graph, Resource type) { + +// System.err.println("affa"); + + graph.asyncRequest(new TypeSetAndString(type), new AsyncProcedure, String>>() { + + @Override + public void execute(AsyncReadGraph graph, Pair, String> typeInfo) { + + Set types = typeInfo.first; + if(types.contains(L0.Ontology)) { + +// dqs.forPossibleRelatedValue(graph, resource, name, serializer, new AsyncProcedure() { +// +// @Override +// public void execute(AsyncReadGraph graph, String result) { +//// if(names++ % 1000 == 0) System.err.println("names=" + names + "(" + result + ")"); +// } +// +// @Override +// public void exception(AsyncReadGraph graph, Throwable throwable) { +// } +// +// }); + + } else if (types.contains(L0.Library)) { + //dqs.forEachObject(graph, resource, consistsOf, procedure); + } + + } + + @Override + public void exception(AsyncReadGraph graph, Throwable throwable) { + } + + }); + + } + + @Override + public void exception(AsyncReadGraph graph, Throwable throwable) { + } + + }); + + + } + + public static Read readAsyncTypes(final Resource resource) { + + return new ReadRequest() { + + @Override + public void run(ReadGraph graph) { + + final Layer0 L0 = Layer0.getInstance(graph); + + try { + +// graph.syncRequest(new TypeSetAndString(L0.Library), new TransientCacheListener, String>>()); +// graph.syncRequest(new TypeSetAndString(L0.Ontology), new TransientCacheListener, String>>()); + + final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class); + final RelationInfo consistsOf = graph.syncRequest(new AsyncRead() { + + @Override + public void perform(AsyncReadGraph graph, AsyncProcedure procedure) { + dqs.forRelationInfo(graph, L0.ConsistsOf, procedure); + } + + @Override + public int threadHash() { + return hashCode(); + } + + @Override + public int getFlags() { + return 0; + } + + }); + final RelationInfo name = graph.syncRequest(new AsyncRead() { + + @Override + public void perform(AsyncReadGraph graph, AsyncProcedure procedure) { + dqs.forRelationInfo(graph, L0.HasName, procedure); + } + + @Override + public int threadHash() { + return hashCode(); + } + + @Override + public int getFlags() { + return 0; + } + + }); + + final Serializer serializer = FastStringSerializer.INSTANCE;//Bindings.getSerializerUnchecked( Bindings.STRING ); + + final AsyncProcedure names = new AsyncProcedure() { + + @Override + public void execute(AsyncReadGraph graph, String name) { +// System.out.println("exec: " + name); + } + + @Override + public void exception(AsyncReadGraph graph, Throwable throwable) { + throwable.printStackTrace(); + } + + }; + +// readAsyncLoop(dqs, graph, resource, new AsyncMultiProcedure() { +// +// @Override +// public void execute(AsyncReadGraph graph, Resource child) { +// readAsyncTypesLoop(dqs, L0, graph, child, consistsOf, this, name, serializer, names); +// } +// +// @Override +// public void finished(AsyncReadGraph graph) { +// } +// +// @Override +// public void exception(AsyncReadGraph graph, Throwable throwable) { +// throwable.printStackTrace(); +// } +// +// }, name, serializer, names); + + } catch (Exception e) { + + e.printStackTrace(); + + } + + } + + }; + + } + + +}