org.simantics.xml.sax/src/org/simantics/xml/data/XmlDataConverter.java

   1 package org.simantics.xml.data;
   2
   3 import java.io.File;
   4 import java.io.FileInputStream;
   5 import java.io.IOException;
   6 import java.util.ArrayDeque;
   7 import java.util.Date;
   8 import java.util.Deque;
   9 import java.util.HashMap;
  10 import java.util.HashSet;
  11 import java.util.Iterator;
  12 import java.util.LinkedHashMap;
  13 import java.util.List;
  14 import java.util.Map;
  15 import java.util.Set;
  16 import java.util.regex.Matcher;
  17
  18 import javax.xml.bind.JAXBContext;
  19 import javax.xml.bind.JAXBElement;
  20 import javax.xml.bind.JAXBException;
  21 import javax.xml.bind.Marshaller;
  22 import javax.xml.namespace.QName;
  23 import javax.xml.stream.XMLEventReader;
  24 import javax.xml.stream.XMLInputFactory;
  25 import javax.xml.stream.XMLStreamException;
  26 import javax.xml.stream.events.Attribute;
  27 import javax.xml.stream.events.Characters;
  28 import javax.xml.stream.events.EndElement;
  29 import javax.xml.stream.events.StartElement;
  30 import javax.xml.stream.events.XMLEvent;
  31
  32 import org.simantics.xml.sax.SchemaConversionBase;
  33 import org.w3._2001.xmlschema.Annotated;
  34 import org.w3._2001.xmlschema.ComplexType;
  35 import org.w3._2001.xmlschema.Element;
  36 import org.w3._2001.xmlschema.ExplicitGroup;
  37 import org.w3._2001.xmlschema.Import;
  38 import org.w3._2001.xmlschema.LocalComplexType;
  39 import org.w3._2001.xmlschema.LocalElement;
  40 import org.w3._2001.xmlschema.OpenAttrs;
  41 import org.w3._2001.xmlschema.Schema;
  42 import org.w3._2001.xmlschema.TopLevelElement;
  43
  44 /**
  45  * This class generates XML-file parsers based on bunch of XML data files. It is recommended to use schema based parser (org.simantics.xml.sax.SchemaConverter) if possible.
  46  * Parser generated by this class is not reliable...
  47  *
  48  * @author luukkainen
  49  *
  50  */
  51 public class XmlDataConverter {
  52
  53         File outputPlugin;
  54         File conversionFile;
  55         List<File> inputFiles;
  56
  57         String pluginName;
  58
  59         private String[] header;
  60
  61         public XmlDataConverter(List<File> inputFiles, File conversionFile, File outputPlugin) {
  62                 if (inputFiles.size() == 0)
  63                         throw new IllegalArgumentException("At least one input file must be given.");
  64                 this.outputPlugin = outputPlugin;
  65                 this.conversionFile = conversionFile;
  66                 this.inputFiles = inputFiles;
  67
  68                 pluginName = outputPlugin.getName();
  69
  70         }
  71
  72         public void convert() throws IOException, XMLStreamException, JAXBException {
  73
  74                 init();
  75                 doConvert();
  76
  77                 Map<Schema, File> fileMap = new HashMap<>();
  78                 JAXBContext jc = JAXBContext.newInstance("org.w3._2001.xmlschema");
  79                 Marshaller m = jc.createMarshaller();
  80                 m.setProperty("jaxb.formatted.output", true);
  81                 Set<String> filenames = new HashSet<>();
  82                 for (Schema s : schemaMap.values()) {
  83                         String name = s.getTargetNamespace();
  84                         // Special case for XAML
  85                         if (name.startsWith("clr-namespace:")) {
  86                                 name = name.substring("clr-namespace:".length());
  87                                 int i = name.indexOf(";assembly");
  88                                 if (i > 0)
  89                                         name = name.substring(0, i);
  90                         }
  91                         name = name.replaceAll("\\.", "_");
  92                         name = name.replaceAll("/", "_");
  93                         name = name.replaceAll(":", "_");
  94                         name = name.replaceAll(";", "_");
  95                         if (filenames.contains(name)) {
  96                                 int i = 2;
  97                                 while (filenames.contains(name+i)) {
  98                                         i++;
  99                                 }
 100                                 name = name+i;
 101                         }
 102                         filenames.add(name);
 103                         File file = new File(outputPlugin.getAbsolutePath() + File.separator + name +".xsd");
 104                         fileMap.put(s, file);
 105                 }
 106                 for (Schema s : schemaMap.values()) {
 107                         for (OpenAttrs openAttrs : s.getIncludeOrImportOrRedefine()) {
 108                                 if (openAttrs instanceof Import) {
 109                                         Import import1 = (Import)openAttrs;
 110                                         Schema dep = schemaMap.get(import1.getNamespace());
 111                                         import1.setSchemaLocation(fileMap.get(dep).getName());
 112                                 }
 113                         }
 114                 }
 115                 for (Schema s : schemaMap.values()) {
 116                         File file = fileMap.get(s);
 117                         m.marshal(s, file);
 118                 }
 119                 Schema rootSchema = schemaMap.values().iterator().next();
 120                 DataSchemaConverter schemaConverter = new DataSchemaConverter(rootSchema,fileMap.get(rootSchema),conversionFile,outputPlugin);
 121                 schemaConverter.setFileMap(fileMap);
 122                 schemaConverter.setSchemaMap(schemaMap);
 123                 schemaConverter.convert();
 124
 125
 126                 header = null;
 127                 schemaMap = null;
 128                 elementMap = null;
 129         }
 130
 131         protected void init()  throws IOException {
 132
 133                 header = new String[4];
 134                 header[0] = "Generated with org.simantics.xml.sax XML data file converter";
 135                 header[1] = "";
 136                 header[2] = "File " + inputFiles.get(0).getAbsolutePath().replaceAll(Matcher.quoteReplacement("\\"), "/") + " , total file count: " + (inputFiles.size()) + "";
 137                 header[3] = "Date " + new Date().toString();
 138
 139                 schemaMap = new HashMap<>();
 140                 elementMap = new HashMap<>();
 141                 attributeMap = new HashMap<>();
 142                 elementNsMap = new HashMap<>();
 143         }
 144
 145         Map<String, Schema> schemaMap = new LinkedHashMap<>();
 146         Map<Schema,Map<String,Element>> elementMap = new HashMap<>();
 147         Map<Element,String> elementNsMap = new HashMap<>();
 148         Map<Schema,Map<String,org.w3._2001.xmlschema.Attribute>> attributeMap = new HashMap<>();
 149
 150         protected void doConvert() throws IOException, XMLStreamException, JAXBException {
 151                 XMLInputFactory input = XMLInputFactory.newInstance();
 152
 153
 154                 for (File inputFile : inputFiles) {
 155                         XMLEventReader reader = input.createXMLEventReader(new FileInputStream(inputFile));
 156                         convertFile(reader);
 157                 }
 158
 159         }
 160
 161         private void convertFile(XMLEventReader reader) throws XMLStreamException {
 162                 Deque<Element> elementStack = new ArrayDeque<>();
 163                 while (reader.hasNext()) {
 164                         XMLEvent event = reader.nextEvent();
 165                         if (event.isStartElement()) {
 166                                 StartElement parseElement = event.asStartElement();
 167 //                              System.out.println("Start " + parseElement.getName());
 168                                 Element schemaElement = null;
 169                                 String currentNS = parseElement.getName().getNamespaceURI();
 170                                 Schema s = schemaMap.get(currentNS);
 171                                 String elementName = parseElement.getName().getLocalPart();
 172                                 if (s == null) {
 173                                         s = getOrCreateSchema(parseElement);
 174                                 } else {
 175                                         schemaElement = elementMap.get(s).get(elementName);
 176                                 }
 177                                 if (elementName.equals("CanvasLayers.TracingLayer") || elementName.equals("Layer"))
 178                                         System.out.println();
 179                                 Element parentElement = elementStack.peek();
 180
 181                                 boolean newElement = false;
 182                                 boolean sameNameSpace = true;
 183
 184                                 if (parentElement != null) {
 185                                         //QName parentType = parentElement.getType();
 186                                         String parentNs = elementNsMap.get(parentElement);
 187                                         sameNameSpace =currentNS.equals(parentNs);
 188                                         if (!sameNameSpace) {
 189                                                 Schema ps = getOrCreateSchema(parentNs);
 190                                                 addSchemaDependency(ps, s);
 191                                         }
 192
 193                                 }
 194                                 if (schemaElement == null) {
 195
 196                                         //QName type = null;
 197                                         if (elementStack.isEmpty()) {
 198                                                 schemaElement = new TopLevelElement();
 199                                                 s.getSimpleTypeOrComplexTypeOrGroup().add(schemaElement);
 200                                         } else {
 201
 202
 203 //                                              if (sameNameSpace) {
 204 //                                                      localElement = new LocalElement();
 205 //                                                      schemaElement = localElement;
 206 //                                                      //type = new QName(elementName);
 207 //
 208 //                                              } else {
 209                                                         schemaElement = new TopLevelElement();
 210                                                         s.getSimpleTypeOrComplexTypeOrGroup().add(schemaElement);
 211                                                         //type = new QName(SchemaConversionBase.SCHEMA_NS,"element");
 212
 213
 214 //                                              }
 215                                         }
 216                                         schemaElement.setName(elementName);
 217                                         elementNsMap.put(schemaElement, currentNS);
 218
 219
 220                                         elementMap.get(s).put(elementName, schemaElement);
 221                                         newElement = true;
 222                                 }
 223                                 if (parentElement != null) {
 224                                         ComplexType complexType = parentElement.getComplexType();
 225                                         ExplicitGroup choice = complexType.getChoice();
 226                                         if (choice == null) {
 227                                                 choice = new ExplicitGroup();
 228                                                 complexType.setChoice(choice);
 229                                                 choice.setMaxOccurs("unbounded");
 230                                         }
 231                                         LocalElement localElement = new LocalElement();
 232                                         localElement.setRef(new QName(parseElement.getName().getNamespaceURI(), elementName));
 233
 234                                         addElement(choice, new QName(SchemaConversionBase.SCHEMA_NS,"element"), localElement);
 235                                 }
 236
 237                                 elementStack.push(schemaElement);
 238
 239                                 Iterator<Attribute> attributeIterator = parseElement.getAttributes();
 240
 241 //                              while (attributeIterator.hasNext()) {
 242 //                                      Attribute attribute = attributeIterator.next();
 243 //                                      System.out.println("Attribute " + attribute.getName() + " " + attribute.getValue());
 244 //                              }
 245                                 if (newElement) {
 246                                         LocalComplexType complexType = new LocalComplexType();
 247                                         schemaElement.setComplexType(complexType);
 248                                         attributeIterator = parseElement.getAttributes();
 249                                         while (attributeIterator.hasNext()) {
 250                                                 Attribute attribute = attributeIterator.next();
 251                                                 if ("http://www.w3.org/XML/1998/namespace".equals(attribute.getName().getNamespaceURI()))
 252                                                         continue;
 253                                                 addAttribute(attribute, complexType, parseElement.getNamespaceURI(attribute.getName().getPrefix()));
 254                                         }
 255
 256                                 } else {
 257                                         LocalComplexType complexType = schemaElement.getComplexType();
 258                                         attributeIterator = parseElement.getAttributes();
 259                                         Map<String,org.w3._2001.xmlschema.Attribute> currentAttributes = new HashMap<>();
 260                                         Iterator<Annotated> currentAttributeIterator = complexType.getAttributeOrAttributeGroup().iterator();
 261                                         while (currentAttributeIterator.hasNext()) {
 262                                                 Annotated annotated = currentAttributeIterator.next();
 263                                                 if (annotated instanceof org.w3._2001.xmlschema.Attribute) {
 264                                                         org.w3._2001.xmlschema.Attribute schemaAttribute = (org.w3._2001.xmlschema.Attribute)annotated;
 265                                                         String n = schemaAttribute.getName();
 266                                                         if (n != null)
 267                                                                 currentAttributes.put(n, schemaAttribute);
 268                                                 }
 269                                         }
 270                                         while (attributeIterator.hasNext()) {
 271                                                 Attribute attribute = attributeIterator.next();
 272                                                 if ("http://www.w3.org/XML/1998/namespace".equals(attribute.getName().getNamespaceURI()))
 273                                                         continue;
 274                                                 org.w3._2001.xmlschema.Attribute schemaAttribute = currentAttributes.get(attribute.getName().getLocalPart());
 275                                                 if (schemaAttribute == null) {
 276                                                         addAttribute(attribute, complexType, parseElement.getNamespaceURI(attribute.getName().getPrefix()));
 277                                                 } else {
 278                                                         QName newType = getType(attribute.getValue());
 279                                                         updateAttributeType(schemaAttribute, newType);
 280                                                 }
 281
 282                                         }
 283                                 }
 284
 285                         } else if (event.isEndElement()) {
 286                                 EndElement element = event.asEndElement();
 287 //                              System.out.println("End " + element.getName());
 288                                 elementStack.pop();
 289                         } else if (event.isAttribute()) {
 290                                 System.out.println(event);
 291                         } else if (event.isStartDocument()) {
 292                                 System.out.println(event);
 293                         } else if (event.isEndDocument()) {
 294
 295                         } else if (event.isEntityReference()) {
 296
 297                         } else if (event.isCharacters()) {
 298                                 Characters characters = event.asCharacters();
 299 //                              if (!characters.isWhiteSpace())
 300 //                                      System.out.println(characters.getData());
 301                     } else if (event.isNamespace()) {
 302                         System.out.println(event);
 303                     }
 304                 }
 305         }
 306
 307         private void updateAttributeType(org.w3._2001.xmlschema.Attribute schemaAttribute, QName newType) {
 308
 309                 QName currentType = schemaAttribute.getType();
 310                 if (!newType.getLocalPart().equals(currentType.getLocalPart())) {
 311
 312
 313                         if (currentType.getLocalPart().equals("integer") && newType.getLocalPart().equals("double")) {
 314                                 // change integer to double
 315                                 schemaAttribute.setType(newType);
 316                         } else if (currentType.getLocalPart().equals("double") && newType.getLocalPart().equals("integer")) {
 317                                 // nothing to do, integer can be parsed as double
 318                         } else if (!currentType.getLocalPart().equals("string")){
 319                                 schemaAttribute.setType(new QName(SchemaConversionBase.SCHEMA_NS, "string"));
 320                         }
 321                 }
 322         }
 323
 324         private void addElement(ExplicitGroup choice, QName type, LocalElement localElement) {
 325                 for (Object o  : choice.getParticle()) {
 326                         JAXBElement<LocalElement> el = (JAXBElement<LocalElement>)o;
 327                         if (el.getName().equals(type)) {
 328                                 QName ref = el.getValue().getRef();
 329                                 QName ref2 = localElement.getRef();
 330                                 if (ref != null) {
 331                                         if (ref.equals(ref2))
 332                                                 return;
 333                                 } else if (el.getValue().getType().equals(localElement.getType()))
 334                                                 return;
 335                         }
 336
 337                 }
 338                 choice.getParticle().add(new JAXBElement<LocalElement>(type, LocalElement.class, null, localElement));
 339         }
 340
 341         private void addSchemaDependency(Schema parentSchema, Schema schema) {
 342                 for (OpenAttrs openAttrs : parentSchema.getIncludeOrImportOrRedefine()) {
 343                         if (openAttrs instanceof Import) {
 344                                 Import import1 = (Import)openAttrs;
 345                                 if (import1.getNamespace().equals(schema.getTargetNamespace()))
 346                                         return;
 347                         }
 348                 }
 349                 Import import1 = new Import();
 350                 import1.setNamespace(schema.getTargetNamespace());
 351                 parentSchema.getIncludeOrImportOrRedefine().add(import1);
 352         }
 353
 354         private void addAttribute(Attribute attribute, ComplexType complexType, String currentNS) {
 355                 if (attribute.getName().getLocalPart().equals("Panel.ZIndex"))
 356                         System.out.println();
 357                 if (attribute.getName().getNamespaceURI().length() == 0 || attribute.getName().getNamespaceURI().equals(currentNS)) {
 358                         org.w3._2001.xmlschema.Attribute schemaAttribute = new org.w3._2001.xmlschema.Attribute();
 359                         schemaAttribute.setName(attribute.getName().getLocalPart());
 360                         schemaAttribute.setType(getType(attribute.getValue()));
 361                         addAttribute(complexType, schemaAttribute);
 362                 } else {
 363                         {
 364                                 Schema schema = getOrCreateSchema(currentNS);
 365                                 Schema attrSchema = getOrCreateSchema(attribute.getName().getNamespaceURI());
 366
 367                                 org.w3._2001.xmlschema.Attribute schemaAttribute = attributeMap.get(attrSchema).get(attribute.getName().getLocalPart());
 368                                 if (schemaAttribute == null) {
 369                                         schemaAttribute = new org.w3._2001.xmlschema.TopLevelAttribute();
 370                                         schemaAttribute.setName(attribute.getName().getLocalPart());
 371                                         schemaAttribute.setType(getType(attribute.getValue()));
 372                                         attrSchema.getSimpleTypeOrComplexTypeOrGroup().add(schemaAttribute);
 373                                         attributeMap.get(attrSchema).put(attribute.getName().getLocalPart(), schemaAttribute);
 374                                 }
 375                                 addSchemaDependency(schema, attrSchema);
 376
 377                         }
 378                         {
 379                                 org.w3._2001.xmlschema.Attribute schemaAttribute = new org.w3._2001.xmlschema.Attribute();
 380                                 schemaAttribute.setRef(new QName(attribute.getName().getNamespaceURI(),attribute.getName().getLocalPart()));
 381                                 addAttribute(complexType, schemaAttribute);
 382                         }
 383
 384                 }
 385         }
 386
 387         private void addAttribute(ComplexType complexType, org.w3._2001.xmlschema.Attribute schemaAttribute) {
 388                 if (schemaAttribute.getName() != null) {
 389                         for (Annotated annotated : complexType.getAttributeOrAttributeGroup()) {
 390                                 if (annotated instanceof org.w3._2001.xmlschema.Attribute) {
 391                                         org.w3._2001.xmlschema.Attribute attr = (org.w3._2001.xmlschema.Attribute)annotated;
 392                                         if (schemaAttribute.getName().equals(attr.getName())) {
 393                                                 updateAttributeType(attr, schemaAttribute.getType());
 394                                         }
 395                                 }
 396                         }
 397                 } else {
 398                         for (Annotated annotated : complexType.getAttributeOrAttributeGroup()) {
 399                                 if (annotated instanceof org.w3._2001.xmlschema.Attribute) {
 400                                         org.w3._2001.xmlschema.Attribute attr = (org.w3._2001.xmlschema.Attribute)annotated;
 401                                         if (attr.getName() != null)
 402                                                 continue;
 403                                         if (schemaAttribute.getRef().equals(attr.getRef())) {
 404                                                 return;
 405                                         }
 406                                 }
 407                         }
 408                 }
 409                 if ("Panel.ZIndex".equals(schemaAttribute.getName()))
 410                         System.out.println();
 411                 complexType.getAttributeOrAttributeGroup().add(schemaAttribute);
 412         }
 413
 414
 415         private QName getType(String value) {
 416                 try {
 417                         Integer.parseInt(value);
 418                         return new QName(SchemaConversionBase.SCHEMA_NS, "integer");
 419                 } catch (NumberFormatException e) {
 420
 421                 }
 422
 423                 try {
 424                         Double.parseDouble(value);
 425                         return new QName(SchemaConversionBase.SCHEMA_NS, "double");
 426                 } catch (NumberFormatException e) {
 427
 428                 }
 429                 if ("True".equals(value) || "False".equals(value))
 430                         return new QName(SchemaConversionBase.SCHEMA_NS, "boolean");
 431                 return new QName(SchemaConversionBase.SCHEMA_NS, "string");
 432
 433         }
 434
 435         private Schema getOrCreateSchema(StartElement parseElement) {
 436                 return getOrCreateSchema(parseElement.getName().getNamespaceURI());
 437         }
 438
 439         private Schema getOrCreateSchema(String ns) {
 440                 if (ns == null)
 441                         throw new IllegalArgumentException("Schema NS cannot be null.");
 442                 Schema s = schemaMap.get(ns);
 443                 if (s == null) {
 444                         s = new Schema();
 445                         s.setTargetNamespace(ns);
 446                         schemaMap.put(ns, s);
 447                         elementMap.put(s, new HashMap<String,Element>());
 448                         attributeMap.put(s, new HashMap<String, org.w3._2001.xmlschema.Attribute>());
 449                 }
 450                 return s;
 451         }
 452
 453 }