org.simantics.xml.sax/src/org/simantics/xml/data/XmlDataConverter.java

   1 package org.simantics.xml.data;
   2
   3 import java.io.File;
   4 import java.io.FileInputStream;
   5 import java.io.IOException;
   6 import java.util.ArrayDeque;
   7 import java.util.Date;
   8 import java.util.Deque;
   9 import java.util.HashMap;
  10 import java.util.HashSet;
  11 import java.util.Iterator;
  12 import java.util.LinkedHashMap;
  13 import java.util.List;
  14 import java.util.Map;
  15 import java.util.Set;
  16 import java.util.regex.Matcher;
  17
  18 import javax.xml.bind.JAXBContext;
  19 import javax.xml.bind.JAXBElement;
  20 import javax.xml.bind.JAXBException;
  21 import javax.xml.bind.Marshaller;
  22 import javax.xml.namespace.QName;
  23 import javax.xml.stream.XMLEventReader;
  24 import javax.xml.stream.XMLInputFactory;
  25 import javax.xml.stream.XMLStreamException;
  26 import javax.xml.stream.events.Attribute;
  27 import javax.xml.stream.events.Characters;
  28 import javax.xml.stream.events.EndElement;
  29 import javax.xml.stream.events.StartElement;
  30 import javax.xml.stream.events.XMLEvent;
  31
  32 import org.simantics.xml.sax.SchemaConversionBase;
  33 import org.w3._2001.xmlschema.Annotated;
  34 import org.w3._2001.xmlschema.ComplexType;
  35 import org.w3._2001.xmlschema.Element;
  36 import org.w3._2001.xmlschema.ExplicitGroup;
  37 import org.w3._2001.xmlschema.Import;
  38 import org.w3._2001.xmlschema.LocalComplexType;
  39 import org.w3._2001.xmlschema.LocalElement;
  40 import org.w3._2001.xmlschema.OpenAttrs;
  41 import org.w3._2001.xmlschema.Schema;
  42 import org.w3._2001.xmlschema.TopLevelElement;
  43
  44 /**
  45  * This class generates XML-file parsers based on bunch of XML data files. It is recommended to use schema based parser (org.simantics.xml.sax.SchemaConverter) if possible.
  46  * Parser generated by this class is not reliable...
  47  *
  48  * @author luukkainen
  49  *
  50  */
  51 public class XmlDataConverter {
  52
  53         File outputPlugin;
  54         File conversionFile;
  55         List<File> inputFiles;
  56
  57         String pluginName;
  58
  59         private String[] header;
  60
  61         public XmlDataConverter(List<File> inputFiles, File conversionFile, File outputPlugin) {
  62                 if (inputFiles.size() == 0)
  63                         throw new IllegalArgumentException("At least one input file must be given.");
  64                 this.outputPlugin = outputPlugin;
  65                 this.conversionFile = conversionFile;
  66                 this.inputFiles = inputFiles;
  67
  68                 pluginName = outputPlugin.getName();
  69
  70         }
  71
  72         public void convert() throws IOException, XMLStreamException, JAXBException {
  73
  74                 init();
  75                 doConvert();
  76
  77                 Map<Schema, File> fileMap = new HashMap<>();
  78                 JAXBContext jc = JAXBContext.newInstance("org.w3._2001.xmlschema");
  79                 Marshaller m = jc.createMarshaller();
  80                 m.setProperty("jaxb.formatted.output", true);
  81                 Set<String> filenames = new HashSet<>();
  82                 for (Schema s : schemaMap.values()) {
  83                         String name = s.getTargetNamespace();
  84                         // Special case for XAML
  85                         if (name.startsWith("clr-namespace:")) {
  86                                 name = name.substring("clr-namespace:".length());
  87                                 int i = name.indexOf(";assembly");
  88                                 if (i > 0)
  89                                         name = name.substring(0, i);
  90                         }
  91                         name = name.replaceAll("\\.", "_");
  92                         name = name.replaceAll("/", "_");
  93                         name = name.replaceAll(":", "_");
  94                         name = name.replaceAll(";", "_");
  95                         if (filenames.contains(name)) {
  96                                 int i = 2;
  97                                 while (filenames.contains(name+i)) {
  98                                         i++;
  99                                 }
 100                                 name = name+i;
 101                         }
 102                         filenames.add(name);
 103                         File file = new File(outputPlugin.getAbsolutePath() + File.separator + name +".xsd");
 104                         fileMap.put(s, file);
 105                 }
 106                 for (Schema s : schemaMap.values()) {
 107                         for (OpenAttrs openAttrs : s.getIncludeOrImportOrRedefine()) {
 108                                 if (openAttrs instanceof Import) {
 109                                         Import import1 = (Import)openAttrs;
 110                                         Schema dep = schemaMap.get(import1.getNamespace());
 111                                         import1.setSchemaLocation(fileMap.get(dep).getName());
 112                                 }
 113                         }
 114                 }
 115                 for (Schema s : schemaMap.values()) {
 116                         File file = fileMap.get(s);
 117                         m.marshal(s, file);
 118                 }
 119                 Schema rootSchema = schemaMap.values().iterator().next();
 120                 DataSchemaConverter schemaConverter = new DataSchemaConverter(rootSchema,fileMap.get(rootSchema),conversionFile,outputPlugin);
 121                 schemaConverter.setFileMap(fileMap);
 122                 schemaConverter.setSchemaMap(schemaMap);
 123                 schemaConverter.convert();
 124
 125
 126                 header = null;
 127                 schemaMap = null;
 128                 elementMap = null;
 129         }
 130
 131         protected void init()  throws IOException {
 132
 133                 header = new String[4];
 134                 header[0] = "Generated with org.simantics.xml.sax XML data file converter";
 135                 header[1] = "";
 136                 header[2] = "File " + inputFiles.get(0).getAbsolutePath().replaceAll(Matcher.quoteReplacement("\\"), "/") + " , total file count: " + (inputFiles.size()) + "";
 137                 header[3] = "Date " + new Date().toString();
 138
 139                 schemaMap = new HashMap<>();
 140                 elementMap = new HashMap<>();
 141                 attributeMap = new HashMap<>();
 142                 elementNsMap = new HashMap<>();
 143         }
 144
 145         Map<String, Schema> schemaMap = new LinkedHashMap<>();
 146         Map<Schema,Map<String,Element>> elementMap = new HashMap<>();
 147         Map<Element,String> elementNsMap = new HashMap<>();
 148         Map<Schema,Map<String,org.w3._2001.xmlschema.Attribute>> attributeMap = new HashMap<>();
 149
 150         protected void doConvert() throws IOException, XMLStreamException, JAXBException {
 151                 XMLInputFactory input = XMLInputFactory.newInstance();
 152
 153
 154                 for (File inputFile : inputFiles) {
 155                         XMLEventReader reader = input.createXMLEventReader(new FileInputStream(inputFile));
 156                         convertFile(reader);
 157                 }
 158
 159         }
 160
 161         private void convertFile(XMLEventReader reader) throws XMLStreamException {
 162                 Deque<Element> elementStack = new ArrayDeque<>();
 163                 while (reader.hasNext()) {
 164                         XMLEvent event = reader.nextEvent();
 165                         if (event.isStartElement()) {
 166                                 StartElement parseElement = event.asStartElement();
 167 //                              System.out.println("Start " + parseElement.getName());
 168                                 Element schemaElement = null;
 169                                 String currentNS = parseElement.getName().getNamespaceURI();
 170                                 Schema s = schemaMap.get(currentNS);
 171                                 String elementName = parseElement.getName().getLocalPart();
 172                                 if (s == null) {
 173                                         s = getOrCreateSchema(parseElement);
 174                                 } else {
 175                                         schemaElement = elementMap.get(s).get(elementName);
 176                                 }
 177                                 Element parentElement = elementStack.peek();
 178
 179                                 boolean newElement = false;
 180                                 boolean sameNameSpace = true;
 181
 182                                 if (parentElement != null) {
 183                                         String parentNs = elementNsMap.get(parentElement);
 184                                         sameNameSpace =currentNS.equals(parentNs);
 185                                         if (!sameNameSpace) {
 186                                                 Schema ps = getOrCreateSchema(parentNs);
 187                                                 addSchemaDependency(ps, s);
 188                                         }
 189
 190                                 }
 191                                 if (schemaElement == null) {
 192                                         if (elementStack.isEmpty()) {
 193                                                 schemaElement = new TopLevelElement();
 194                                                 s.getSimpleTypeOrComplexTypeOrGroup().add(schemaElement);
 195                                         } else {
 196                                                 schemaElement = new TopLevelElement();
 197                                                 s.getSimpleTypeOrComplexTypeOrGroup().add(schemaElement);
 198                                         }
 199                                         schemaElement.setName(elementName);
 200                                         elementNsMap.put(schemaElement, currentNS);
 201
 202
 203                                         elementMap.get(s).put(elementName, schemaElement);
 204                                         newElement = true;
 205                                 }
 206                                 if (parentElement != null) {
 207                                         ComplexType complexType = parentElement.getComplexType();
 208                                         ExplicitGroup choice = complexType.getChoice();
 209                                         if (choice == null) {
 210                                                 choice = new ExplicitGroup();
 211                                                 complexType.setChoice(choice);
 212                                                 choice.setMaxOccurs("unbounded");
 213                                         }
 214                                         LocalElement localElement = new LocalElement();
 215                                         localElement.setRef(new QName(parseElement.getName().getNamespaceURI(), elementName));
 216
 217                                         addElement(choice, new QName(SchemaConversionBase.SCHEMA_NS,"element"), localElement);
 218                                 }
 219
 220                                 elementStack.push(schemaElement);
 221
 222                                 Iterator<Attribute> attributeIterator = parseElement.getAttributes();
 223
 224 //                              while (attributeIterator.hasNext()) {
 225 //                                      Attribute attribute = attributeIterator.next();
 226 //                                      System.out.println("Attribute " + attribute.getName() + " " + attribute.getValue());
 227 //                              }
 228                                 if (newElement) {
 229                                         LocalComplexType complexType = new LocalComplexType();
 230                                         schemaElement.setComplexType(complexType);
 231                                         attributeIterator = parseElement.getAttributes();
 232                                         while (attributeIterator.hasNext()) {
 233                                                 Attribute attribute = attributeIterator.next();
 234                                                 if ("http://www.w3.org/XML/1998/namespace".equals(attribute.getName().getNamespaceURI()))
 235                                                         continue;
 236                                                 addAttribute(attribute, complexType, currentNS);
 237                                         }
 238
 239                                 } else {
 240                                         LocalComplexType complexType = schemaElement.getComplexType();
 241                                         attributeIterator = parseElement.getAttributes();
 242                                         Map<String,org.w3._2001.xmlschema.Attribute> currentAttributes = new HashMap<>();
 243                                         Iterator<Annotated> currentAttributeIterator = complexType.getAttributeOrAttributeGroup().iterator();
 244                                         while (currentAttributeIterator.hasNext()) {
 245                                                 Annotated annotated = currentAttributeIterator.next();
 246                                                 if (annotated instanceof org.w3._2001.xmlschema.Attribute) {
 247                                                         org.w3._2001.xmlschema.Attribute schemaAttribute = (org.w3._2001.xmlschema.Attribute)annotated;
 248                                                         String n = schemaAttribute.getName();
 249                                                         if (n != null)
 250                                                                 currentAttributes.put(n, schemaAttribute);
 251                                                 }
 252                                         }
 253                                         while (attributeIterator.hasNext()) {
 254                                                 Attribute attribute = attributeIterator.next();
 255                                                 if ("http://www.w3.org/XML/1998/namespace".equals(attribute.getName().getNamespaceURI()))
 256                                                         continue;
 257                                                 org.w3._2001.xmlschema.Attribute schemaAttribute = currentAttributes.get(attribute.getName().getLocalPart());
 258                                                 if (schemaAttribute == null) {
 259                                                         addAttribute(attribute, complexType, currentNS);
 260                                                 } else {
 261                                                         QName newType = getType(attribute.getValue());
 262                                                         updateAttributeType(schemaAttribute, newType);
 263                                                 }
 264
 265                                         }
 266                                 }
 267
 268                         } else if (event.isEndElement()) {
 269                                 EndElement element = event.asEndElement();
 270 //                              System.out.println("End " + element.getName());
 271                                 elementStack.pop();
 272                         } else if (event.isAttribute()) {
 273
 274                         } else if (event.isStartDocument()) {
 275
 276                         } else if (event.isEndDocument()) {
 277
 278                         } else if (event.isEntityReference()) {
 279
 280                         } else if (event.isCharacters()) {
 281                                 Characters characters = event.asCharacters();
 282 //                              if (!characters.isWhiteSpace())
 283 //                                      System.out.println(characters.getData());
 284                     } else if (event.isNamespace()) {
 285
 286                     }
 287                 }
 288         }
 289
 290         private void updateAttributeType(org.w3._2001.xmlschema.Attribute schemaAttribute, QName newType) {
 291
 292                 QName currentType = schemaAttribute.getType();
 293                 if (!newType.getLocalPart().equals(currentType.getLocalPart())) {
 294
 295
 296                         if (currentType.getLocalPart().equals("integer") && newType.getLocalPart().equals("double")) {
 297                                 // change integer to double
 298                                 schemaAttribute.setType(newType);
 299                         } else if (currentType.getLocalPart().equals("double") && newType.getLocalPart().equals("integer")) {
 300                                 // nothing to do, integer can be parsed as double
 301                         } else if (!currentType.getLocalPart().equals("string")){
 302                                 schemaAttribute.setType(new QName(SchemaConversionBase.SCHEMA_NS, "string"));
 303                         }
 304                 }
 305         }
 306
 307         private void addElement(ExplicitGroup choice, QName type, LocalElement localElement) {
 308                 for (Object o  : choice.getParticle()) {
 309                         JAXBElement<LocalElement> el = (JAXBElement<LocalElement>)o;
 310                         if (el.getName().equals(type)) {
 311                                 QName ref = el.getValue().getRef();
 312                                 QName ref2 = localElement.getRef();
 313                                 if (ref != null) {
 314                                         if (ref.equals(ref2))
 315                                                 return;
 316                                 } else if (el.getValue().getType().equals(localElement.getType()))
 317                                                 return;
 318                         }
 319
 320                 }
 321                 choice.getParticle().add(new JAXBElement<LocalElement>(type, LocalElement.class, null, localElement));
 322         }
 323
 324         private void addSchemaDependency(Schema parentSchema, Schema schema) {
 325                 for (OpenAttrs openAttrs : parentSchema.getIncludeOrImportOrRedefine()) {
 326                         if (openAttrs instanceof Import) {
 327                                 Import import1 = (Import)openAttrs;
 328                                 if (import1.getNamespace().equals(schema.getTargetNamespace()))
 329                                         return;
 330                         }
 331                 }
 332                 Import import1 = new Import();
 333                 import1.setNamespace(schema.getTargetNamespace());
 334                 parentSchema.getIncludeOrImportOrRedefine().add(import1);
 335         }
 336
 337         private void addAttribute(Attribute attribute, ComplexType complexType, String currentNS) {
 338                 if (attribute.getName().getNamespaceURI().length() == 0 || attribute.getName().getNamespaceURI().equals(currentNS)) {
 339                         org.w3._2001.xmlschema.Attribute schemaAttribute = new org.w3._2001.xmlschema.Attribute();
 340                         schemaAttribute.setName(attribute.getName().getLocalPart());
 341                         schemaAttribute.setType(getType(attribute.getValue()));
 342                         addAttribute(complexType, schemaAttribute);
 343                 } else {
 344                         {
 345                                 Schema schema = getOrCreateSchema(currentNS);
 346                                 Schema attrSchema = getOrCreateSchema(attribute.getName().getNamespaceURI());
 347
 348                                 org.w3._2001.xmlschema.Attribute schemaAttribute = attributeMap.get(attrSchema).get(attribute.getName().getLocalPart());
 349                                 if (schemaAttribute == null) {
 350                                         schemaAttribute = new org.w3._2001.xmlschema.TopLevelAttribute();
 351                                         schemaAttribute.setName(attribute.getName().getLocalPart());
 352                                         schemaAttribute.setType(getType(attribute.getValue()));
 353                                         attrSchema.getSimpleTypeOrComplexTypeOrGroup().add(schemaAttribute);
 354                                         attributeMap.get(attrSchema).put(attribute.getName().getLocalPart(), schemaAttribute);
 355                                 }
 356                                 addSchemaDependency(schema, attrSchema);
 357
 358                         }
 359                         {
 360                                 org.w3._2001.xmlschema.Attribute schemaAttribute = new org.w3._2001.xmlschema.Attribute();
 361                                 schemaAttribute.setRef(new QName(attribute.getName().getNamespaceURI(),attribute.getName().getLocalPart()));
 362                                 addAttribute(complexType, schemaAttribute);
 363                         }
 364
 365                 }
 366         }
 367
 368         private void addAttribute(ComplexType complexType, org.w3._2001.xmlschema.Attribute schemaAttribute) {
 369                 if (schemaAttribute.getName() != null) {
 370                         for (Annotated annotated : complexType.getAttributeOrAttributeGroup()) {
 371                                 if (annotated instanceof org.w3._2001.xmlschema.Attribute) {
 372                                         org.w3._2001.xmlschema.Attribute attr = (org.w3._2001.xmlschema.Attribute)annotated;
 373                                         if (schemaAttribute.getName().equals(attr.getName())) {
 374                                                 updateAttributeType(attr, schemaAttribute.getType());
 375                                         }
 376                                 }
 377                         }
 378                 } else {
 379                         for (Annotated annotated : complexType.getAttributeOrAttributeGroup()) {
 380                                 if (annotated instanceof org.w3._2001.xmlschema.Attribute) {
 381                                         org.w3._2001.xmlschema.Attribute attr = (org.w3._2001.xmlschema.Attribute)annotated;
 382                                         if (attr.getName() != null)
 383                                                 continue;
 384                                         if (schemaAttribute.getRef().equals(attr.getRef())) {
 385                                                 return;
 386                                         }
 387                                 }
 388                         }
 389                 }
 390                 complexType.getAttributeOrAttributeGroup().add(schemaAttribute);
 391         }
 392
 393
 394         private QName getType(String value) {
 395                 try {
 396                         Integer.parseInt(value);
 397                         return new QName(SchemaConversionBase.SCHEMA_NS, "integer");
 398                 } catch (NumberFormatException e) {
 399
 400                 }
 401
 402                 try {
 403                         Double.parseDouble(value);
 404                         return new QName(SchemaConversionBase.SCHEMA_NS, "double");
 405                 } catch (NumberFormatException e) {
 406
 407                 }
 408                 if ("True".equals(value) || "False".equals(value))
 409                         return new QName(SchemaConversionBase.SCHEMA_NS, "boolean");
 410                 return new QName(SchemaConversionBase.SCHEMA_NS, "string");
 411
 412         }
 413
 414         private Schema getOrCreateSchema(StartElement parseElement) {
 415                 return getOrCreateSchema(parseElement.getName().getNamespaceURI());
 416         }
 417
 418         private Schema getOrCreateSchema(String ns) {
 419                 if (ns == null)
 420                         throw new IllegalArgumentException("Schema NS cannot be null.");
 421                 Schema s = schemaMap.get(ns);
 422                 if (s == null) {
 423                         s = new Schema();
 424                         s.setTargetNamespace(ns);
 425                         schemaMap.put(ns, s);
 426                         elementMap.put(s, new HashMap<String,Element>());
 427                         attributeMap.put(s, new HashMap<String, org.w3._2001.xmlschema.Attribute>());
 428                 }
 429                 return s;
 430         }
 431
 432 }