From: luukkainen Date: Thu, 2 May 2013 10:49:45 +0000 (+0000) Subject: refs #3842 X-Git-Tag: v1.31.0~72 X-Git-Url: https://gerrit.simantics.org/r/gitweb?a=commitdiff_plain;h=c3f03da51d12fc81d9779829e909dff9a0673ef7;p=simantics%2Finterop.git refs #3842 git-svn-id: https://www.simantics.org/svn/simantics/interoperability/trunk@27302 ac1ea38d-2e2b-0410-8846-a27921b304fc --- diff --git a/org.simantics.interop.xmlio/META-INF/MANIFEST.MF b/org.simantics.interop.xmlio/META-INF/MANIFEST.MF index 762d755..a5c223a 100644 --- a/org.simantics.interop.xmlio/META-INF/MANIFEST.MF +++ b/org.simantics.interop.xmlio/META-INF/MANIFEST.MF @@ -10,7 +10,8 @@ Require-Bundle: org.eclipse.ui, org.simantics.db;bundle-version="1.1.0", org.simantics.db.common;bundle-version="1.1.0", org.apache.xerces;bundle-version="2.9.0", - org.simantics.db.layer0;bundle-version="1.1.0" + org.simantics.db.layer0;bundle-version="1.1.0", + org.cojen;bundle-version="2.2.1" Bundle-RequiredExecutionEnvironment: JavaSE-1.6 Bundle-ActivationPolicy: lazy Export-Package: org.simantics.interop.xmlio diff --git a/org.simantics.interop.xmlio/src/org/simantics/interop/xmlio/LoadXML.java b/org.simantics.interop.xmlio/src/org/simantics/interop/xmlio/LoadXML.java index 3a6606b..c79ace6 100644 --- a/org.simantics.interop.xmlio/src/org/simantics/interop/xmlio/LoadXML.java +++ b/org.simantics.interop.xmlio/src/org/simantics/interop/xmlio/LoadXML.java @@ -6,14 +6,13 @@ import java.io.FileReader; import java.io.Reader; import java.io.Serializable; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Stack; import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; +import org.cojen.util.IntHashMap; import org.simantics.db.ReadGraph; import org.simantics.db.Resource; import org.simantics.db.Session; @@ -21,6 +20,7 @@ import org.simantics.db.WriteGraph; import org.simantics.db.common.request.DelayedWriteRequest; import org.simantics.db.common.request.WriteRequest; import org.simantics.db.exception.DatabaseException; +import org.simantics.db.layer0.util.Layer0Utils; import org.simantics.db.layer0.util.SessionGarbageCollection; import org.simantics.layer0.Layer0; import org.xml.sax.Attributes; @@ -38,7 +38,9 @@ public class LoadXML { private File file; - private Map idMap = new HashMap(); + + //private Map idMap = new HashMap(); + private IntHashMap idMap = new IntHashMap(); //private Map uriMap = new HashMap(); @@ -49,6 +51,8 @@ public class LoadXML { private int resources; private int statements; + private static final boolean USE_DELAYED = true; + public LoadXML(Session session, File file) { this.session = session; this.file = file; @@ -59,33 +63,68 @@ public class LoadXML { this.rule = rule; } - public void load(final Resource subj, final Resource pred) throws Exception { + public void load(Resource subj, Resource pred) throws Exception { resources = 0; statements = 0; - // TODO : switch to Delayed if possible - // Delayed Write Graph in 1.6 does not seem to support converting virtual Resources to real Resources! - session.syncRequest(new DelayedWriteRequest() { - //session.syncRequest(new WriteRequest() { - - @Override - public void perform(WriteGraph graph) throws DatabaseException { - LoadXML.this.graph = graph; - l0 = Layer0.getInstance(graph); - try { - XMLParser parser = new XMLParser(); - parser.parse(file.getAbsolutePath()); - } catch (Exception e) { - throw new DatabaseException(e); - } finally { - resultRoot = idMap.get(0); - graph.claim(subj, pred, resultRoot); - idMap.clear(); - } - } - }); + if (USE_DELAYED) { + session.syncRequest(new ImportDelayedRequest(subj,pred)); + } else { + session.syncRequest(new ImportRequest(subj,pred)); + } + //session.syncRequest(new DelayedWriteRequest() { + + + System.out.println("Import done."); + } + + private class ImportRequest extends WriteRequest { + Resource subj;Resource pred; + + + public ImportRequest(Resource subj, Resource pred) { + super(); + this.subj = subj; + this.pred = pred; + } + + @Override + public void perform(WriteGraph graph) throws DatabaseException { + doImport(graph, subj, pred); + } + } + + private class ImportDelayedRequest extends DelayedWriteRequest { + Resource subj;Resource pred; - System.out.println("Imported " + statements + " Statements, " + resources + " Resources."); + + public ImportDelayedRequest(Resource subj, Resource pred) { + super(); + this.subj = subj; + this.pred = pred; + } + + @Override + public void perform(WriteGraph graph) throws DatabaseException { + doImport(graph, subj, pred); + } + } + + private void doImport(WriteGraph graph, Resource subj, Resource pred) throws DatabaseException { + LoadXML.this.graph = graph; + l0 = Layer0.getInstance(graph); + Layer0Utils.setDependenciesIndexingDisabled(graph, true); + try { + XMLParser parser = new XMLParser(); + parser.parse(file.getAbsolutePath()); + System.out.println("Imported " + statements + " Statements, " + resources + " Resources."); + } catch (Exception e) { + throw new DatabaseException(e); + } finally { + resultRoot = idMap.get(0); + graph.claim(subj, pred, resultRoot); + idMap.clear(); + } } @@ -242,6 +281,11 @@ public class LoadXML { System.out.println("Imported " + statements + " Statements, " + resources + " Resources.."); //SessionGarbageCollection.gc(graph, 0, -1); } + if (!USE_DELAYED && statements % 100000 == 0) { + SessionGarbageCollection.gc(graph, 0, -1); + System.gc(); + } + return; } diff --git a/org.simantics.interop.xmlio/src/org/simantics/interop/xmlio/MigrateLoadRule.java b/org.simantics.interop.xmlio/src/org/simantics/interop/xmlio/MigrateLoadRule.java new file mode 100644 index 0000000..e0d5fdc --- /dev/null +++ b/org.simantics.interop.xmlio/src/org/simantics/interop/xmlio/MigrateLoadRule.java @@ -0,0 +1,68 @@ +package org.simantics.interop.xmlio; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.simantics.db.ReadGraph; +import org.simantics.db.Resource; +import org.simantics.db.exception.DatabaseException; +import org.simantics.layer0.Layer0; + +public class MigrateLoadRule extends DefaultLoadRule { + + Set ontologies = new HashSet(); + Map ontologyVersions = new HashMap(); + + public MigrateLoadRule(ReadGraph graph) throws DatabaseException{ + Resource root = graph.getSession().getRootLibrary(); + Layer0 l0 = Layer0.getInstance(graph); + for (Resource lib : graph.getObjects(root, l0.ConsistsOf)) { + if (!graph.isInstanceOf(lib, l0.Library)) + continue; + for (Resource ontology : graph.getObjects(lib, l0.ConsistsOf)) { + if (!graph.isInstanceOf(ontology, l0.Ontology)) + continue; + String uri = graph.getURI(ontology); + String parsed[] = parse(uri); + ontologies.add(parsed[0]); + ontologyVersions.put(parsed[0], parsed[1]); + } + } + + } + @Override + public Resource getResource(ReadGraph graph, String uri) throws DatabaseException { + return super.getResource(graph, getUri(uri)); + } + + //Map mappedUris = new HashMap(); + + private String getUri(String uri) { + + String parsed[] = parse(uri); + String currentUri = parsed[0] + "-" + ontologyVersions.get(parsed[0]) + "/" + parsed[2]; + return currentUri; + } + + private String[] parse(String uri) { + int startVer = uri.indexOf("-"); + int endVer = uri.indexOf("/", startVer); + if (endVer == -1) { + return new String[]{uri.substring(0,startVer),uri.substring(startVer+1)}; + } else { + return new String[]{uri.substring(0,startVer),uri.substring(startVer+1,endVer),uri.substring(endVer+1)}; + } + + } + + @Override + public Object createValue(ReadGraph graph, List value, List types) throws DatabaseException { + for (int i = 0; i < types.size(); i++) { + types.set(i, getUri(types.get(i))); + } + return super.createValue(graph, value, types); + } +}