org.simantics.interop.xmlio/src/org/simantics/interop/xmlio/LoadXML.java

   1 package org.simantics.interop.xmlio;\r
   2 \r
   3 import java.io.BufferedReader;\r
   4 import java.io.File;\r
   5 import java.io.FileReader;\r
   6 import java.io.Reader;\r
   7 import java.io.Serializable;\r
   8 import java.util.ArrayList;\r
   9 import java.util.List;\r
  10 import java.util.Stack;\r
  11 \r
  12 import javax.xml.parsers.SAXParser;\r
  13 import javax.xml.parsers.SAXParserFactory;\r
  14 \r
  15 import org.cojen.util.IntHashMap;\r
  16 import org.simantics.db.ReadGraph;\r
  17 import org.simantics.db.Resource;\r
  18 import org.simantics.db.Session;\r
  19 import org.simantics.db.WriteGraph;\r
  20 import org.simantics.db.common.request.DelayedWriteRequest;\r
  21 import org.simantics.db.common.request.WriteRequest;\r
  22 import org.simantics.db.exception.DatabaseException;\r
  23 import org.simantics.db.layer0.util.Layer0Utils;\r
  24 import org.simantics.db.layer0.util.SessionGarbageCollection;\r
  25 import org.simantics.layer0.Layer0;\r
  26 import org.xml.sax.Attributes;\r
  27 import org.xml.sax.InputSource;\r
  28 import org.xml.sax.SAXException;\r
  29 import org.xml.sax.XMLReader;\r
  30 import org.xml.sax.helpers.DefaultHandler;\r
  31 \r
  32 \r
  33 \r
  34 public class LoadXML {\r
  35 \r
  36         private Session session;\r
  37         private Resource resultRoot;\r
  38         \r
  39         private File file;\r
  40         \r
  41         \r
  42         //private Map<Integer,Resource> idMap = new HashMap<Integer, Resource>();\r
  43         private IntHashMap<Resource> idMap = new IntHashMap<Resource>();\r
  44         \r
  45         //private Map<String,Resource> uriMap = new HashMap<String, Resource>();\r
  46         \r
  47         private WriteGraph graph;\r
  48         private Layer0 l0;\r
  49         private LoadRule rule;\r
  50         \r
  51         private int resources;\r
  52         private int statements;\r
  53         \r
  54         private static final boolean USE_DELAYED = true;\r
  55         \r
  56         public LoadXML(Session session, File file) {\r
  57                 this.session = session;\r
  58                 this.file = file;\r
  59                 this.rule = new DefaultLoadRule();\r
  60         }\r
  61         \r
  62         public void setRule(LoadRule rule) {\r
  63                 this.rule = rule;\r
  64         }\r
  65         \r
  66         public void load(Resource subj, Resource pred) throws Exception {\r
  67                 resources = 0;\r
  68                 statements = 0;\r
  69                 \r
  70                 if (USE_DELAYED) {\r
  71                         session.syncRequest(new ImportDelayedRequest(subj,pred));       \r
  72                 } else {\r
  73                         session.syncRequest(new ImportRequest(subj,pred));      \r
  74                 }\r
  75                 //session.syncRequest(new DelayedWriteRequest() {\r
  76                 \r
  77                 \r
  78                 System.out.println("Import done.");\r
  79         }\r
  80         \r
  81         private class ImportRequest extends WriteRequest {\r
  82                 Resource subj;Resource pred;\r
  83                 \r
  84                 \r
  85                 public ImportRequest(Resource subj, Resource pred) {\r
  86                         super();\r
  87                         this.subj = subj;\r
  88                         this.pred = pred;\r
  89                 }\r
  90 \r
  91                 @Override\r
  92                 public void perform(WriteGraph graph) throws DatabaseException {\r
  93                         doImport(graph, subj, pred);\r
  94                 }\r
  95         }\r
  96         \r
  97         private class ImportDelayedRequest extends DelayedWriteRequest {\r
  98                 Resource subj;Resource pred;\r
  99                 \r
 100                 \r
 101                 public ImportDelayedRequest(Resource subj, Resource pred) {\r
 102                         super();\r
 103                         this.subj = subj;\r
 104                         this.pred = pred;\r
 105                 }\r
 106 \r
 107                 @Override\r
 108                 public void perform(WriteGraph graph) throws DatabaseException {\r
 109                         doImport(graph, subj, pred);\r
 110                 }\r
 111         }\r
 112         \r
 113         private void doImport(WriteGraph graph, Resource subj, Resource pred) throws DatabaseException {\r
 114                 LoadXML.this.graph = graph;\r
 115                 l0 = Layer0.getInstance(graph);\r
 116                 Layer0Utils.setDependenciesIndexingDisabled(graph, true);\r
 117                 try {\r
 118                         XMLParser parser = new XMLParser();\r
 119                         parser.parse(file.getAbsolutePath());\r
 120                         System.out.println("Imported " + statements + " Statements, " + resources + " Resources.");\r
 121                 } catch (Exception e) {\r
 122                         throw new DatabaseException(e);\r
 123                 } finally {\r
 124                         resultRoot = idMap.get(0);\r
 125                         graph.claim(subj, pred, resultRoot);\r
 126                         idMap.clear();\r
 127                 }\r
 128         }\r
 129         \r
 130         \r
 131 //      public Resource getResult() {\r
 132 //              return resultRoot;\r
 133 //      }\r
 134         \r
 135         protected Resource getResource(ReadGraph graph, String uri) throws DatabaseException {\r
 136                 return rule.getResource(graph, uri);\r
 137         }\r
 138         \r
 139         protected Object createValue(ReadGraph graph, List<String> value, List<String> types) throws DatabaseException {\r
 140                 return rule.createValue(graph, value, types);\r
 141         }\r
 142         \r
 143         protected void initializeWithBundles(ReadGraph graph, List<Bundle> bundles) throws DatabaseException {\r
 144                 rule.initializeWithBundles(graph, bundles);\r
 145         }\r
 146         \r
 147         \r
 148         public class XMLParser  extends DefaultHandler implements Serializable {\r
 149 \r
 150                 private static final long serialVersionUID = -4995836637014958966L;\r
 151                 \r
 152                 private Stack<Element> current = new Stack<Element>();\r
 153                 \r
 154                 private void loadElement(Element element) throws SAXException{\r
 155                         String name = element.qName;\r
 156                         if ("graphexport".equals(name))\r
 157                                 return;\r
 158                         if ("graphbundles".equals(name)) {\r
 159                                 element.setData(new ArrayList<Bundle>());\r
 160                                 return;\r
 161                         }\r
 162                         if ("bundle".equals(name))\r
 163                                 return;\r
 164                         if ("resource".equals(name)) {\r
 165                                 element.setData(new Res());\r
 166                                 return;\r
 167                         }\r
 168                         if ("type".equals(name)) {\r
 169                                 return;\r
 170                         }\r
 171                         if ("statement".equals(name))\r
 172                                 return;\r
 173                         \r
 174                 }\r
 175                 \r
 176                 private void handleElement(Stack<Element> parents, Element element) throws SAXException{\r
 177                         try {\r
 178                         String elementName = element.qName;\r
 179                         if ("graphexport".equals(elementName))\r
 180                                 return;\r
 181                         if ("graphbundles".equals(elementName)) {\r
 182                                 List<Bundle> list = (List<Bundle>)element.getData();\r
 183                                 initializeWithBundles(graph, list);\r
 184                                 element.setData(null);\r
 185                                 return;\r
 186                         }\r
 187                         if ("bundle".equals(elementName)) {\r
 188                                 List<Bundle> list = (List<Bundle>)parents.peek().getData();\r
 189                                 String name = null;\r
 190                                 String versionId = null;\r
 191                                 for (Attribute a : element.getAttributes()) {\r
 192                                         if ("name".equals(a.qName))\r
 193                                                 name = a.value;\r
 194                                         if ("versionid".equals(a.qName))\r
 195                                                 versionId = a.value;\r
 196                                 } \r
 197                                 list.add(new Bundle(name,versionId));\r
 198                                 return;\r
 199                         }\r
 200                         if ("resource".equals(elementName)) {\r
 201                                 Res res = (Res)element.getData();\r
 202                                 boolean rel = false;\r
 203                                 String uri = null;\r
 204                                 int id = -1;\r
 205                                 Resource r = null;\r
 206                                 for (Attribute a : element.getAttributes()) {\r
 207                                         if ("id".equals(a.qName))\r
 208                                                 id = Integer.parseInt(a.value);\r
 209                                         if ("rel".equals(a.qName))\r
 210                                                 rel = "true".equals(a.value);\r
 211                                         if ("uri".equals(a.qName))\r
 212                                                 uri = a.value;\r
 213                                 } \r
 214                                 if (rel) {\r
 215                                         r = getResource(graph, uri);\r
 216                                 } else {\r
 217                                         \r
 218                                         r = graph.newResource();\r
 219                                         for (String typeUri : res.types) {\r
 220                                                 Resource type =getResource(graph, typeUri);\r
 221                                                 graph.claim(r, l0.InstanceOf, type);\r
 222                                         }\r
 223                                         \r
 224                                         if (res.values.size() > 0 ) {\r
 225                                                 Object valObj = createValue(graph, res.values, res.types);\r
 226                                                 graph.claimValue(r, valObj);\r
 227                                         }\r
 228                                 }\r
 229                                 if (id == -1)\r
 230                                         throw new SAXException("Resource id missing");\r
 231                                 idMap.put(id, r);\r
 232                                 \r
 233                                 res.types.clear();\r
 234                                 res.values.clear();\r
 235                                 element.setData(null);\r
 236                                 resources++;\r
 237                                 return;\r
 238                         }\r
 239                         if ("type".equals(elementName)) {\r
 240                                 Element parent = parents.peek();\r
 241                                 Res res = (Res)parent.getData();\r
 242                                 String uri = null;\r
 243                                 for (Attribute a : element.getAttributes()) {\r
 244                                         if ("uri".equals(a.qName))\r
 245                                                 uri = a.value;\r
 246                                 } \r
 247                                 if (uri != null)\r
 248                                         res.types.add(uri);\r
 249                                 \r
 250                                 return;\r
 251                         }\r
 252                         if ("value".equals(elementName)) {\r
 253                                 Element parent = parents.peek();\r
 254                                 Res res = (Res)parent.getData();\r
 255                                 String value = null;\r
 256                                 for (Attribute a : element.getAttributes()) {\r
 257                                         if ("value".equals(a.qName))\r
 258                                                 value = a.value;\r
 259                                 } \r
 260                                 if (value != null)\r
 261                                         res.values.add(value);\r
 262                                 \r
 263                                 return;\r
 264                         }\r
 265                         if ("statement".equals(elementName)) {\r
 266                                 int subjId = -1;\r
 267                                 int predId = -1;\r
 268                                 int objId = -1;\r
 269                                 for (Attribute a : element.getAttributes()) {\r
 270                                         if ("subject".equals(a.qName))\r
 271                                                 subjId = Integer.parseInt(a.value);\r
 272                                         if ("predicate".equals(a.qName))\r
 273                                                 predId = Integer.parseInt(a.value);\r
 274                                         if ("object".equals(a.qName))\r
 275                                                 objId = Integer.parseInt(a.value);\r
 276                                 } \r
 277                                 \r
 278                                 graph.claim(idMap.get(subjId), idMap.get(predId), idMap.get(objId));\r
 279                                 statements++;\r
 280                                 if (statements % 10000 == 0) {\r
 281                                         System.out.println("Imported " + statements + " Statements, " + resources + " Resources..");\r
 282                                         //SessionGarbageCollection.gc(graph, 0, -1);\r
 283                                 }\r
 284                                 if (!USE_DELAYED && statements % 100000 == 0) {\r
 285                                         SessionGarbageCollection.gc(graph, 0, -1);\r
 286                                         System.gc();\r
 287                                 }\r
 288                                 \r
 289                                 return;\r
 290                         }\r
 291                         \r
 292                         } catch (DatabaseException e) {\r
 293                                 throw new SAXException(e);\r
 294                         }\r
 295                 }\r
 296                 \r
 297                 public void done() {\r
 298                         \r
 299                 }\r
 300                 \r
 301                 public void parse(String filename) throws Exception {\r
 302                         SAXParserFactory spf = SAXParserFactory.newInstance();\r
 303                         SAXParser saxParser = spf.newSAXParser();\r
 304                         \r
 305                         XMLReader reader = saxParser.getXMLReader();\r
 306                         Reader file = new BufferedReader(new FileReader(filename));\r
 307                         reader.setContentHandler(this);\r
 308                         reader.parse(new InputSource(file));\r
 309                         done();\r
 310                 }\r
 311                 \r
 312                 @Override\r
 313                 public void startElement(String uri, String localName, String name,\r
 314                                 Attributes attributes) throws SAXException {\r
 315                         Element e = new Element(uri,localName,name,attributes);\r
 316                         current.push(e);\r
 317                         //texts.push(new String());\r
 318                         loadElement(e);\r
 319                 }\r
 320                 \r
 321                 @Override\r
 322                 public void endElement(String uri, String localName, String name)\r
 323                                 throws SAXException {\r
 324                         Element e = null;\r
 325                         if (!current.empty()) {\r
 326                                 e = current.pop();\r
 327                         }\r
 328                         if (e != null) {\r
 329                                 handleElement(current,e);\r
 330                         }\r
 331                 }\r
 332                 \r
 333         }\r
 334         \r
 335         public class Attribute {\r
 336                 public String localName;\r
 337                 public String qName;\r
 338                 public String uri;\r
 339                 public String value;\r
 340                 \r
 341                 public Attribute(String localName, String qName, String uri, String value) {\r
 342                         this.localName = localName;\r
 343                         this.qName = qName;\r
 344                         this.uri = uri;\r
 345                         this.value = value;\r
 346                 }\r
 347         }\r
 348         \r
 349         public class Element implements Serializable {\r
 350                 private static final long serialVersionUID = -5207502156942818875L;\r
 351                 String uri;\r
 352                 String localName;\r
 353                 String qName;\r
 354                 List<Attribute> attributes = new ArrayList<Attribute>();\r
 355                 \r
 356                 public Element(String uri, String localName, String qName, Attributes attributes) {\r
 357                         this.uri = uri;\r
 358                         this.localName = localName;\r
 359                         this.qName = qName;\r
 360                         for (int i = 0; i < attributes.getLength(); i++) {\r
 361                                 this.attributes.add(new Attribute(attributes.getLocalName(i),attributes.getQName(i),attributes.getURI(i),attributes.getValue(i)));\r
 362                         }\r
 363                 }\r
 364 \r
 365                 public String getUri() {\r
 366                         return uri;\r
 367                 }\r
 368 \r
 369                 public String getLocalName() {\r
 370                         return localName;\r
 371                 }\r
 372 \r
 373                 public String getQName() {\r
 374                         return qName;\r
 375                 }\r
 376 \r
 377                 public List<Attribute> getAttributes() {\r
 378                         return attributes;\r
 379                 }\r
 380 \r
 381                 private Object data;\r
 382                 \r
 383                 public void setData(Object data) {\r
 384                         this.data = data;\r
 385                 }\r
 386                 \r
 387                 public Object getData() {\r
 388                         return data;\r
 389                 }\r
 390 \r
 391         }\r
 392         class Res {\r
 393                 \r
 394                 List<String> types = new ArrayList<String>();\r
 395                 List<String> values = new ArrayList<String>();\r
 396         }\r
 397         \r
 398         public static class Bundle {\r
 399                 String name;\r
 400                 String versionId;\r
 401                 public Bundle(String name, String versionId) {\r
 402                         super();\r
 403                         this.name = name;\r
 404                         this.versionId = versionId;\r
 405                 }\r
 406                 \r
 407                 public String getName() {\r
 408                         return name;\r
 409                 }\r
 410                 \r
 411                 public String getVersionId() {\r
 412                         return versionId;\r
 413                 }\r
 414         }\r
 415         \r
 416         \r
 417 }\r