org.simantics.interop.xmlio/src/org/simantics/interop/xmlio/LoadXML.java

   1 package org.simantics.interop.xmlio;
   2
   3 import java.io.BufferedReader;
   4 import java.io.File;
   5 import java.io.FileReader;
   6 import java.io.Reader;
   7 import java.io.Serializable;
   8 import java.util.ArrayList;
   9 import java.util.List;
  10 import java.util.Stack;
  11
  12 import javax.xml.parsers.SAXParser;
  13 import javax.xml.parsers.SAXParserFactory;
  14
  15 import org.simantics.db.ReadGraph;
  16 import org.simantics.db.Resource;
  17 import org.simantics.db.Session;
  18 import org.simantics.db.WriteGraph;
  19 import org.simantics.db.common.request.DelayedWriteRequest;
  20 import org.simantics.db.common.request.WriteRequest;
  21 import org.simantics.db.exception.DatabaseException;
  22 import org.simantics.db.layer0.util.Layer0Utils;
  23 import org.simantics.db.layer0.util.SessionGarbageCollection;
  24 import org.simantics.layer0.Layer0;
  25 import org.xml.sax.Attributes;
  26 import org.xml.sax.InputSource;
  27 import org.xml.sax.SAXException;
  28 import org.xml.sax.XMLReader;
  29 import org.xml.sax.helpers.DefaultHandler;
  30
  31 import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
  32
  33
  34
  35 public class LoadXML {
  36
  37         private Session session;
  38         private Resource resultRoot;
  39
  40         private File file;
  41
  42
  43         //private Map<Integer,Resource> idMap = new HashMap<Integer, Resource>();
  44         Int2ObjectOpenHashMap<Resource> idMap = new Int2ObjectOpenHashMap<Resource>();
  45
  46         //private Map<String,Resource> uriMap = new HashMap<String, Resource>();
  47
  48         private WriteGraph graph;
  49         private Layer0 l0;
  50         private LoadRule rule;
  51
  52         private int resources;
  53         private int statements;
  54
  55         private static final boolean USE_DELAYED = true;
  56
  57         public LoadXML(Session session, File file) {
  58                 this.session = session;
  59                 this.file = file;
  60                 this.rule = new DefaultLoadRule();
  61         }
  62
  63         public void setRule(LoadRule rule) {
  64                 this.rule = rule;
  65         }
  66
  67         public void load(Resource subj, Resource pred) throws Exception {
  68                 resources = 0;
  69                 statements = 0;
  70
  71                 if (USE_DELAYED) {
  72                         session.syncRequest(new ImportDelayedRequest(subj,pred));
  73                 } else {
  74                         session.syncRequest(new ImportRequest(subj,pred));
  75                 }
  76                 //session.syncRequest(new DelayedWriteRequest() {
  77
  78
  79                 System.out.println("Import done.");
  80         }
  81
  82         private class ImportRequest extends WriteRequest {
  83                 Resource subj;Resource pred;
  84
  85
  86                 public ImportRequest(Resource subj, Resource pred) {
  87                         super();
  88                         this.subj = subj;
  89                         this.pred = pred;
  90                 }
  91
  92                 @Override
  93                 public void perform(WriteGraph graph) throws DatabaseException {
  94                         doImport(graph, subj, pred);
  95                 }
  96         }
  97
  98         private class ImportDelayedRequest extends DelayedWriteRequest {
  99                 Resource subj;Resource pred;
 100
 101
 102                 public ImportDelayedRequest(Resource subj, Resource pred) {
 103                         super();
 104                         this.subj = subj;
 105                         this.pred = pred;
 106                 }
 107
 108                 @Override
 109                 public void perform(WriteGraph graph) throws DatabaseException {
 110                         doImport(graph, subj, pred);
 111                 }
 112         }
 113
 114         private void doImport(WriteGraph graph, Resource subj, Resource pred) throws DatabaseException {
 115                 LoadXML.this.graph = graph;
 116                 l0 = Layer0.getInstance(graph);
 117                 Layer0Utils.setDependenciesIndexingDisabled(graph, true);
 118                 try {
 119                         XMLParser parser = new XMLParser();
 120                         parser.parse(file.getAbsolutePath());
 121                         System.out.println("Imported " + statements + " Statements, " + resources + " Resources.");
 122                 } catch (Exception e) {
 123                         throw new DatabaseException(e);
 124                 } finally {
 125                         resultRoot = idMap.get(0);
 126                         graph.claim(subj, pred, resultRoot);
 127                         idMap.clear();
 128                 }
 129         }
 130
 131
 132 //      public Resource getResult() {
 133 //              return resultRoot;
 134 //      }
 135
 136         protected Resource getResource(ReadGraph graph, String uri) throws DatabaseException {
 137                 return rule.getResource(graph, uri);
 138         }
 139
 140         protected Object createValue(ReadGraph graph, List<String> value, List<String> types) throws DatabaseException {
 141                 return rule.createValue(graph, value, types);
 142         }
 143
 144         protected void initializeWithBundles(ReadGraph graph, List<Bundle> bundles) throws DatabaseException {
 145                 rule.initializeWithBundles(graph, bundles);
 146         }
 147
 148
 149         public class XMLParser  extends DefaultHandler implements Serializable {
 150
 151                 private static final long serialVersionUID = -4995836637014958966L;
 152
 153                 private Stack<Element> current = new Stack<Element>();
 154
 155                 private void loadElement(Element element) throws SAXException{
 156                         String name = element.qName;
 157                         if ("graphexport".equals(name))
 158                                 return;
 159                         if ("graphbundles".equals(name)) {
 160                                 element.setData(new ArrayList<Bundle>());
 161                                 return;
 162                         }
 163                         if ("bundle".equals(name))
 164                                 return;
 165                         if ("resource".equals(name)) {
 166                                 element.setData(new Res());
 167                                 return;
 168                         }
 169                         if ("type".equals(name)) {
 170                                 return;
 171                         }
 172                         if ("statement".equals(name))
 173                                 return;
 174
 175                 }
 176
 177                 private void handleElement(Stack<Element> parents, Element element) throws SAXException{
 178                         try {
 179                         String elementName = element.qName;
 180                         if ("graphexport".equals(elementName))
 181                                 return;
 182                         if ("graphbundles".equals(elementName)) {
 183                                 List<Bundle> list = (List<Bundle>)element.getData();
 184                                 initializeWithBundles(graph, list);
 185                                 element.setData(null);
 186                                 return;
 187                         }
 188                         if ("bundle".equals(elementName)) {
 189                                 List<Bundle> list = (List<Bundle>)parents.peek().getData();
 190                                 String name = null;
 191                                 String versionId = null;
 192                                 for (Attribute a : element.getAttributes()) {
 193                                         if ("name".equals(a.qName))
 194                                                 name = a.value;
 195                                         if ("versionid".equals(a.qName))
 196                                                 versionId = a.value;
 197                                 }
 198                                 list.add(new Bundle(name,versionId));
 199                                 return;
 200                         }
 201                         if ("resource".equals(elementName)) {
 202                                 Res res = (Res)element.getData();
 203                                 boolean rel = false;
 204                                 String uri = null;
 205                                 int id = -1;
 206                                 Resource r = null;
 207                                 for (Attribute a : element.getAttributes()) {
 208                                         if ("id".equals(a.qName))
 209                                                 id = Integer.parseInt(a.value);
 210                                         if ("rel".equals(a.qName))
 211                                                 rel = "true".equals(a.value);
 212                                         if ("uri".equals(a.qName))
 213                                                 uri = a.value;
 214                                 }
 215                                 if (rel) {
 216                                         r = getResource(graph, uri);
 217                                 } else {
 218
 219                                         r = graph.newResource();
 220                                         for (String typeUri : res.types) {
 221                                                 Resource type =getResource(graph, typeUri);
 222                                                 graph.claim(r, l0.InstanceOf, type);
 223                                         }
 224
 225                                         if (res.values.size() > 0 ) {
 226                                                 Object valObj = createValue(graph, res.values, res.types);
 227                                                 graph.claimValue(r, valObj);
 228                                         }
 229                                 }
 230                                 if (id == -1)
 231                                         throw new SAXException("Resource id missing");
 232                                 idMap.put(id, r);
 233
 234                                 res.types.clear();
 235                                 res.values.clear();
 236                                 element.setData(null);
 237                                 resources++;
 238                                 return;
 239                         }
 240                         if ("type".equals(elementName)) {
 241                                 Element parent = parents.peek();
 242                                 Res res = (Res)parent.getData();
 243                                 String uri = null;
 244                                 for (Attribute a : element.getAttributes()) {
 245                                         if ("uri".equals(a.qName))
 246                                                 uri = a.value;
 247                                 }
 248                                 if (uri != null)
 249                                         res.types.add(uri);
 250
 251                                 return;
 252                         }
 253                         if ("value".equals(elementName)) {
 254                                 Element parent = parents.peek();
 255                                 Res res = (Res)parent.getData();
 256                                 String value = null;
 257                                 for (Attribute a : element.getAttributes()) {
 258                                         if ("value".equals(a.qName))
 259                                                 value = a.value;
 260                                 }
 261                                 if (value != null)
 262                                         res.values.add(value);
 263
 264                                 return;
 265                         }
 266                         if ("statement".equals(elementName)) {
 267                                 int subjId = -1;
 268                                 int predId = -1;
 269                                 int objId = -1;
 270                                 for (Attribute a : element.getAttributes()) {
 271                                         if ("subject".equals(a.qName))
 272                                                 subjId = Integer.parseInt(a.value);
 273                                         if ("predicate".equals(a.qName))
 274                                                 predId = Integer.parseInt(a.value);
 275                                         if ("object".equals(a.qName))
 276                                                 objId = Integer.parseInt(a.value);
 277                                 }
 278
 279                                 graph.claim(idMap.get(subjId), idMap.get(predId), idMap.get(objId));
 280                                 statements++;
 281                                 if (statements % 10000 == 0) {
 282                                         System.out.println("Imported " + statements + " Statements, " + resources + " Resources..");
 283                                         //SessionGarbageCollection.gc(graph, 0, -1);
 284                                 }
 285                                 if (!USE_DELAYED && statements % 100000 == 0) {
 286                                         SessionGarbageCollection.gc(graph, 0, -1);
 287                                         System.gc();
 288                                 }
 289
 290                                 return;
 291                         }
 292
 293                         } catch (DatabaseException e) {
 294                                 throw new SAXException(e);
 295                         }
 296                 }
 297
 298                 public void done() {
 299
 300                 }
 301
 302                 public void parse(String filename) throws Exception {
 303                         SAXParserFactory spf = SAXParserFactory.newInstance();
 304                         SAXParser saxParser = spf.newSAXParser();
 305
 306                         XMLReader reader = saxParser.getXMLReader();
 307                         Reader file = new BufferedReader(new FileReader(filename));
 308                         reader.setContentHandler(this);
 309                         reader.parse(new InputSource(file));
 310                         done();
 311                 }
 312
 313                 @Override
 314                 public void startElement(String uri, String localName, String name,
 315                                 Attributes attributes) throws SAXException {
 316                         Element e = new Element(uri,localName,name,attributes);
 317                         current.push(e);
 318                         //texts.push(new String());
 319                         loadElement(e);
 320                 }
 321
 322                 @Override
 323                 public void endElement(String uri, String localName, String name)
 324                                 throws SAXException {
 325                         Element e = null;
 326                         if (!current.empty()) {
 327                                 e = current.pop();
 328                         }
 329                         if (e != null) {
 330                                 handleElement(current,e);
 331                         }
 332                 }
 333
 334         }
 335
 336         public class Attribute {
 337                 public String localName;
 338                 public String qName;
 339                 public String uri;
 340                 public String value;
 341
 342                 public Attribute(String localName, String qName, String uri, String value) {
 343                         this.localName = localName;
 344                         this.qName = qName;
 345                         this.uri = uri;
 346                         this.value = value;
 347                 }
 348         }
 349
 350         public class Element implements Serializable {
 351                 private static final long serialVersionUID = -5207502156942818875L;
 352                 String uri;
 353                 String localName;
 354                 String qName;
 355                 List<Attribute> attributes = new ArrayList<Attribute>();
 356
 357                 public Element(String uri, String localName, String qName, Attributes attributes) {
 358                         this.uri = uri;
 359                         this.localName = localName;
 360                         this.qName = qName;
 361                         for (int i = 0; i < attributes.getLength(); i++) {
 362                                 this.attributes.add(new Attribute(attributes.getLocalName(i),attributes.getQName(i),attributes.getURI(i),attributes.getValue(i)));
 363                         }
 364                 }
 365
 366                 public String getUri() {
 367                         return uri;
 368                 }
 369
 370                 public String getLocalName() {
 371                         return localName;
 372                 }
 373
 374                 public String getQName() {
 375                         return qName;
 376                 }
 377
 378                 public List<Attribute> getAttributes() {
 379                         return attributes;
 380                 }
 381
 382                 private Object data;
 383
 384                 public void setData(Object data) {
 385                         this.data = data;
 386                 }
 387
 388                 public Object getData() {
 389                         return data;
 390                 }
 391
 392         }
 393         class Res {
 394
 395                 List<String> types = new ArrayList<String>();
 396                 List<String> values = new ArrayList<String>();
 397         }
 398
 399         public static class Bundle {
 400                 String name;
 401                 String versionId;
 402                 public Bundle(String name, String versionId) {
 403                         super();
 404                         this.name = name;
 405                         this.versionId = versionId;
 406                 }
 407
 408                 public String getName() {
 409                         return name;
 410                 }
 411
 412                 public String getVersionId() {
 413                         return versionId;
 414                 }
 415         }
 416
 417
 418 }