1 package org.simantics.xml.data;
4 import java.io.FileInputStream;
5 import java.io.IOException;
6 import java.util.ArrayDeque;
8 import java.util.Deque;
9 import java.util.HashMap;
10 import java.util.HashSet;
11 import java.util.Iterator;
12 import java.util.LinkedHashMap;
13 import java.util.List;
16 import java.util.regex.Matcher;
17 import java.util.regex.Pattern;
19 import javax.xml.bind.JAXBContext;
20 import javax.xml.bind.JAXBElement;
21 import javax.xml.bind.JAXBException;
22 import javax.xml.bind.Marshaller;
23 import javax.xml.namespace.QName;
24 import javax.xml.stream.XMLEventReader;
25 import javax.xml.stream.XMLInputFactory;
26 import javax.xml.stream.XMLStreamException;
27 import javax.xml.stream.events.Attribute;
28 import javax.xml.stream.events.Characters;
29 import javax.xml.stream.events.EndElement;
30 import javax.xml.stream.events.StartElement;
31 import javax.xml.stream.events.XMLEvent;
33 import org.simantics.xml.sax.SchemaConversionBase;
34 import org.w3._2001.xmlschema.Annotated;
35 import org.w3._2001.xmlschema.ComplexType;
36 import org.w3._2001.xmlschema.Element;
37 import org.w3._2001.xmlschema.ExplicitGroup;
38 import org.w3._2001.xmlschema.Import;
39 import org.w3._2001.xmlschema.LocalComplexType;
40 import org.w3._2001.xmlschema.LocalElement;
41 import org.w3._2001.xmlschema.LocalSimpleType;
42 import org.w3._2001.xmlschema.OpenAttrs;
43 import org.w3._2001.xmlschema.Restriction;
44 import org.w3._2001.xmlschema.Schema;
45 import org.w3._2001.xmlschema.SimpleType;
46 import org.w3._2001.xmlschema.TopLevelElement;
49 * This class generates XML-file parsers based on bunch of XML data files. It is recommended to use schema based parser (org.simantics.xml.sax.SchemaConverter) if possible.
50 * Parser generated by this class is not reliable...
55 public class XmlDataConverter {
59 List<File> inputFiles;
63 private boolean useElementNSforAttributes = true; // If true, attributes with undefined namespace are written to the same name space as the element. If false. the attributes are written to the root namespace.
64 private boolean ignoreAttributeNS = false; // Completely ignore attribute namespaces. When true, all attributes are written to the elements.
65 private String ignorePattern = "(\\w)*"; // Ignore attribute namespaces. When pattern exists, and local name of a attribute matches the regexp, the interpreted namespace is ignored.
66 private boolean nonStandardBooleans = false; // Accept non standard boolean values (True, False).
68 private String[] header;
70 public XmlDataConverter(List<File> inputFiles, File conversionFile, File outputPlugin) {
71 if (inputFiles.size() == 0)
72 throw new IllegalArgumentException("At least one input file must be given.");
73 this.outputPlugin = outputPlugin;
74 this.conversionFile = conversionFile;
75 this.inputFiles = inputFiles;
77 pluginName = outputPlugin.getName();
81 public void convert() throws IOException, XMLStreamException, JAXBException {
86 Map<Schema, File> fileMap = new HashMap<>();
87 JAXBContext jc = JAXBContext.newInstance("org.w3._2001.xmlschema");
88 Marshaller m = jc.createMarshaller();
89 m.setProperty("jaxb.formatted.output", true);
90 Set<String> filenames = new HashSet<>();
91 for (Schema s : schemaMap.values()) {
92 String name = s.getTargetNamespace();
93 // Special case for XAML
94 if (name.startsWith("clr-namespace:")) {
95 name = name.substring("clr-namespace:".length());
96 int i = name.indexOf(";assembly");
98 name = name.substring(0, i);
100 name = name.replaceAll("\\.", "_");
101 name = name.replaceAll("/", "_");
102 name = name.replaceAll(":", "_");
103 name = name.replaceAll(";", "_");
104 if (filenames.contains(name)) {
106 while (filenames.contains(name+i)) {
112 File file = new File(outputPlugin.getAbsolutePath() + File.separator + name +".xsd");
113 fileMap.put(s, file);
115 for (Schema s : schemaMap.values()) {
116 for (OpenAttrs openAttrs : s.getIncludeOrImportOrRedefine()) {
117 if (openAttrs instanceof Import) {
118 Import import1 = (Import)openAttrs;
119 Schema dep = schemaMap.get(import1.getNamespace());
120 import1.setSchemaLocation(fileMap.get(dep).getName());
124 for (Schema s : schemaMap.values()) {
125 File file = fileMap.get(s);
128 Schema rootSchema = schemaMap.values().iterator().next();
129 DataSchemaConverter schemaConverter = new DataSchemaConverter(rootSchema,fileMap.get(rootSchema),conversionFile,outputPlugin);
130 schemaConverter.setFileMap(fileMap);
131 schemaConverter.setSchemaMap(schemaMap);
132 schemaConverter.convert();
140 protected void init() throws IOException {
142 header = new String[4];
143 header[0] = "Generated with org.simantics.xml.sax XML data file converter";
145 header[2] = "File " + inputFiles.get(0).getAbsolutePath().replaceAll(Matcher.quoteReplacement("\\"), "/") + " , total file count: " + (inputFiles.size()) + "";
146 header[3] = "Date " + new Date().toString();
148 schemaMap = new HashMap<>();
149 elementMap = new HashMap<>();
150 attributeMap = new HashMap<>();
151 elementNsMap = new HashMap<>();
154 Map<String, Schema> schemaMap = new LinkedHashMap<>();
155 Map<Schema,Map<String,Element>> elementMap = new HashMap<>();
156 Map<Element,String> elementNsMap = new HashMap<>();
157 Map<Schema,Map<String,org.w3._2001.xmlschema.Attribute>> attributeMap = new HashMap<>();
160 protected void doConvert() throws IOException, XMLStreamException, JAXBException {
161 XMLInputFactory input = XMLInputFactory.newInstance();
164 for (File inputFile : inputFiles) {
165 XMLEventReader reader = input.createXMLEventReader(new FileInputStream(inputFile));
171 private void convertFile(XMLEventReader reader) throws XMLStreamException {
172 Deque<Element> elementStack = new ArrayDeque<>();
173 while (reader.hasNext()) {
174 XMLEvent event = reader.nextEvent();
175 if (event.isStartElement()) {
176 StartElement parseElement = event.asStartElement();
178 Element schemaElement = null;
179 String currentNS = parseElement.getName().getNamespaceURI();
180 Schema s = schemaMap.get(currentNS);
181 String elementName = parseElement.getName().getLocalPart();
183 s = getOrCreateSchema(parseElement);
185 schemaElement = elementMap.get(s).get(elementName);
188 Element parentElement = elementStack.peek();
190 boolean newElement = false;
191 boolean sameNameSpace = true;
193 if (parentElement != null) {
194 String parentNs = elementNsMap.get(parentElement);
195 sameNameSpace =currentNS.equals(parentNs);
196 if (!sameNameSpace) {
197 Schema ps = getOrCreateSchema(parentNs);
198 addSchemaDependency(ps, s);
202 defaultNS = currentNS;
204 if (schemaElement == null) {
205 if (elementStack.isEmpty()) {
206 schemaElement = new TopLevelElement();
207 s.getSimpleTypeOrComplexTypeOrGroup().add(schemaElement);
209 schemaElement = new TopLevelElement();
210 s.getSimpleTypeOrComplexTypeOrGroup().add(schemaElement);
212 schemaElement.setName(elementName);
213 elementNsMap.put(schemaElement, currentNS);
216 elementMap.get(s).put(elementName, schemaElement);
219 if (parentElement != null) {
220 ComplexType complexType = getOrCreateComplexType(parentElement);
221 ExplicitGroup choice = complexType.getChoice();
222 if (choice == null) {
223 choice = new ExplicitGroup();
224 complexType.setChoice(choice);
225 choice.setMaxOccurs("unbounded");
227 LocalElement localElement = new LocalElement();
228 localElement.setRef(new QName(parseElement.getName().getNamespaceURI(), elementName));
230 addElement(choice, new QName(SchemaConversionBase.SCHEMA_NS,"element"), localElement);
233 elementStack.push(schemaElement);
235 Iterator<Attribute> attributeIterator = parseElement.getAttributes();
237 // while (attributeIterator.hasNext()) {
238 // Attribute attribute = attributeIterator.next();
239 // System.out.println("Attribute " + attribute.getName() + " " + attribute.getValue());
242 attributeIterator = parseElement.getAttributes();
243 if (attributeIterator.hasNext()) {
244 ComplexType complexType = getOrCreateComplexType(schemaElement);
245 while (attributeIterator.hasNext()) {
246 Attribute attribute = attributeIterator.next();
247 if ("http://www.w3.org/XML/1998/namespace".equals(attribute.getName().getNamespaceURI()))
249 addAttribute(attribute, complexType, currentNS);
254 ComplexType complexType = schemaElement.getComplexType();
255 attributeIterator = parseElement.getAttributes();
256 if (complexType != null || attributeIterator.hasNext()) {
257 complexType = getOrCreateComplexType(schemaElement);
258 Map<String,org.w3._2001.xmlschema.Attribute> currentAttributes = new HashMap<>();
259 Iterator<Annotated> currentAttributeIterator = complexType.getAttributeOrAttributeGroup().iterator();
260 while (currentAttributeIterator.hasNext()) {
261 Annotated annotated = currentAttributeIterator.next();
262 if (annotated instanceof org.w3._2001.xmlschema.Attribute) {
263 org.w3._2001.xmlschema.Attribute localAttribute = (org.w3._2001.xmlschema.Attribute)annotated;
264 String n = localAttribute.getName();
266 currentAttributes.put(n, localAttribute);
267 else if (localAttribute.getRef() != null) {
268 currentAttributes.put(localAttribute.getRef().getLocalPart(), localAttribute);
272 while (attributeIterator.hasNext()) {
273 Attribute attribute = attributeIterator.next();
274 if ("http://www.w3.org/XML/1998/namespace".equals(attribute.getName().getNamespaceURI()))
276 org.w3._2001.xmlschema.Attribute localAttribute = currentAttributes.get(attribute.getName().getLocalPart());
277 if (localAttribute == null) {
278 addAttribute(attribute, complexType, currentNS);
280 QName newType = getType(attribute.getValue());
281 org.w3._2001.xmlschema.Attribute schemaAttribute = updateAttributeType(localAttribute, newType);
283 String attrNs = getNS(attribute, currentNS);
284 if (!ignoreAttributeNs(attribute) && attribute.getName().getNamespaceURI().length() > 0) {
285 // Attribute has explicit ns definition.
286 if (localAttribute.getRef() != null) {
287 // current local attribute is reference, check that the namespaces match
288 if (!localAttribute.getRef().getNamespaceURI().equals(attrNs))
289 throw new RuntimeException("Conflicting namespaces for attribute " + attribute.getName().getLocalPart() + " " + attrNs + " " + localAttribute.getRef().getNamespaceURI());
290 } else if (!attrNs.equals(currentNS)){
291 // move the attribute to explicit namespace.
292 complexType.getAttributeOrAttributeGroup().remove(localAttribute);
293 org.w3._2001.xmlschema.Attribute scAttribute = addAttribute(attribute, complexType, currentNS);
294 scAttribute.setType(schemaAttribute.getType());
304 } else if (event.isEndElement()) {
305 EndElement element = event.asEndElement();
306 // System.out.println("End " + element.getName());
308 } else if (event.isAttribute()) {
310 } else if (event.isStartDocument()) {
312 } else if (event.isEndDocument()) {
314 } else if (event.isEntityReference()) {
316 } else if (event.isCharacters()) {
317 Characters characters = event.asCharacters();
318 Element element = elementStack.peek();
319 if (element != null) {
320 String text = characters.getData().trim();
321 if (text.length() > 0) {
322 setElementCharactersData(element, text, characters);
325 } else if (event.isNamespace()) {
331 private void setElementCharactersData(Element element, String text, Characters characters) {
333 //System.out.println(element.getName() + " " + characters.getData());
334 if (element.getComplexType() != null)
335 element.getComplexType().setMixed(true);
337 SimpleType simpleType = getOrCreateSimpleType(element);
338 QName type = getType(text);
339 Restriction restriction = simpleType.getRestriction();
340 if (restriction == null) {
341 restriction = new Restriction();
342 restriction.setBase(type);
343 simpleType.setRestriction(restriction);
345 restriction.setBase(mergePrimitiveType(restriction.getBase(), type));
350 private ComplexType getOrCreateComplexType(Element element) {
351 LocalComplexType complexType = element.getComplexType();
352 if (complexType == null) {
353 complexType = new LocalComplexType();
354 element.setComplexType(complexType);
355 if (element.getSimpleType() != null) {
356 // Convert SimpleType to ComplexType
357 element.setSimpleType(null);
358 complexType.setMixed(true);
364 private SimpleType getOrCreateSimpleType(Element element) {
365 LocalSimpleType simpleType = element.getSimpleType();
366 if (simpleType == null) {
367 simpleType = new LocalSimpleType();
368 element.setSimpleType(simpleType);
373 private void addElement(ExplicitGroup choice, QName type, LocalElement localElement) {
374 for (Object o : choice.getParticle()) {
375 JAXBElement<LocalElement> el = (JAXBElement<LocalElement>)o;
376 if (el.getName().equals(type)) {
377 QName ref = el.getValue().getRef();
378 QName ref2 = localElement.getRef();
380 if (ref.equals(ref2))
382 } else if (el.getValue().getType().equals(localElement.getType()))
387 choice.getParticle().add(new JAXBElement<LocalElement>(type, LocalElement.class, null, localElement));
390 private void addSchemaDependency(Schema parentSchema, Schema schema) {
391 for (OpenAttrs openAttrs : parentSchema.getIncludeOrImportOrRedefine()) {
392 if (openAttrs instanceof Import) {
393 Import import1 = (Import)openAttrs;
394 if (import1.getNamespace().equals(schema.getTargetNamespace()))
398 Import import1 = new Import();
399 import1.setNamespace(schema.getTargetNamespace());
400 parentSchema.getIncludeOrImportOrRedefine().add(import1);
403 private String getNS(Attribute attribute, String currentNS) {
404 if (ignoreAttributeNs(attribute))
406 String attrNs = attribute.getName().getNamespaceURI();
407 if (attrNs.length() == 0) {
408 if (useElementNSforAttributes)
418 private boolean ignoreAttributeNs(Attribute attribute) {
419 if (ignoreAttributeNS)
423 if (ignorePattern != null) {
424 return attribute.getName().getLocalPart().matches(ignorePattern);
430 private org.w3._2001.xmlschema.Attribute addAttribute(Attribute attribute, ComplexType complexType, String currentNS) {
431 String attrNs = getNS(attribute, currentNS);
432 String attrName = attribute.getName().getLocalPart();
434 if (attrNs.equals(currentNS)) {
435 org.w3._2001.xmlschema.Attribute schemaAttribute = new org.w3._2001.xmlschema.Attribute();
436 schemaAttribute.setName(attrName);
437 schemaAttribute.setType(getType(attribute.getValue()));
438 addAttribute(complexType, schemaAttribute);
439 return schemaAttribute;
442 Schema schema = getOrCreateSchema(currentNS);
443 Schema attrSchema = getOrCreateSchema(attrNs);
445 org.w3._2001.xmlschema.Attribute schemaAttribute = attributeMap.get(attrSchema).get(attrName);
446 if (schemaAttribute == null) {
447 schemaAttribute = new org.w3._2001.xmlschema.TopLevelAttribute();
448 schemaAttribute.setName(attrName);
449 schemaAttribute.setType(getType(attribute.getValue()));
450 attrSchema.getSimpleTypeOrComplexTypeOrGroup().add(schemaAttribute);
451 attributeMap.get(attrSchema).put(attribute.getName().getLocalPart(), schemaAttribute);
453 addSchemaDependency(schema, attrSchema);
456 org.w3._2001.xmlschema.Attribute localAttribute = new org.w3._2001.xmlschema.Attribute();
457 localAttribute.setRef(new QName(attrNs,attrName));
458 addAttribute(complexType, localAttribute);
459 return schemaAttribute;
465 private void addAttribute(ComplexType complexType, org.w3._2001.xmlschema.Attribute schemaAttribute) {
466 if (schemaAttribute.getName() != null) {
467 for (Annotated annotated : complexType.getAttributeOrAttributeGroup()) {
468 if (annotated instanceof org.w3._2001.xmlschema.Attribute) {
469 org.w3._2001.xmlschema.Attribute attr = (org.w3._2001.xmlschema.Attribute)annotated;
470 if (schemaAttribute.getName().equals(attr.getName())) {
471 updateAttributeType(attr, schemaAttribute.getType());
477 for (Annotated annotated : complexType.getAttributeOrAttributeGroup()) {
478 if (annotated instanceof org.w3._2001.xmlschema.Attribute) {
479 org.w3._2001.xmlschema.Attribute attr = (org.w3._2001.xmlschema.Attribute)annotated;
480 if (attr.getName() != null)
482 if (schemaAttribute.getRef().equals(attr.getRef())) {
488 complexType.getAttributeOrAttributeGroup().add(schemaAttribute);
492 private QName getType(String value) {
493 if (!nonStandardBooleans) {
494 if ("true".equals(value) || "false".equals(value)) // || "1".equals(value) || "0".equals(value))
495 return new QName(SchemaConversionBase.SCHEMA_NS, "boolean");
497 if ("true".equalsIgnoreCase(value) || "false".equalsIgnoreCase(value)) {
498 return new QName(SchemaConversionBase.SCHEMA_NS, "boolean");
502 Integer.parseInt(value);
503 return new QName(SchemaConversionBase.SCHEMA_NS, "integer");
504 } catch (NumberFormatException e) {
509 Double.parseDouble(value);
510 return new QName(SchemaConversionBase.SCHEMA_NS, "double");
511 } catch (NumberFormatException e) {
515 return new QName(SchemaConversionBase.SCHEMA_NS, "string");
519 private org.w3._2001.xmlschema.Attribute updateAttributeType(org.w3._2001.xmlschema.Attribute schemaAttribute, QName newType) {
521 QName currentType = schemaAttribute.getType();
522 if (currentType == null && schemaAttribute.getRef() != null) {
523 Schema schema = schemaMap.get(schemaAttribute.getRef().getNamespaceURI());
525 schemaAttribute = attributeMap.get(schema).get(schemaAttribute.getRef().getLocalPart());
526 currentType = schemaAttribute.getType();
528 if (currentType == null)
529 throw new RuntimeException("Could not resolve attribute");
532 schemaAttribute.setType(mergePrimitiveType(currentType, newType));
533 return schemaAttribute;
537 private QName mergePrimitiveType(QName currentType, QName newType) {
538 if (!newType.getLocalPart().equals(currentType.getLocalPart())) {
541 if (currentType.getLocalPart().equals("integer") && newType.getLocalPart().equals("double")) {
542 // change integer to double
544 } else if (currentType.getLocalPart().equals("double") && newType.getLocalPart().equals("integer")) {
545 // nothing to do, integer can be parsed as double
547 } else if (currentType.getLocalPart().equals("boolean") && newType.getLocalPart().equals("integer")) {
548 // change boolean to int
550 } else if (currentType.getLocalPart().equals("integer") && newType.getLocalPart().equals("boolean")) {
551 // nothing to do, boolean (0 & 1) can be parsed as integer.
552 // FIXME : what about true / false? Now type detection system does not accept 0 or 1 as boolean values, while XML supports it. See method: QName getType(String value)
553 // FIXME : also, if we support 0 & 1 as booleans, we need to consider possible double values as well.
555 } else if (!currentType.getLocalPart().equals("string")){
556 return new QName(SchemaConversionBase.SCHEMA_NS, "string");
562 private Schema getOrCreateSchema(StartElement parseElement) {
563 return getOrCreateSchema(parseElement.getName().getNamespaceURI());
566 private Schema getOrCreateSchema(String ns) {
568 throw new IllegalArgumentException("Schema NS cannot be null.");
569 Schema s = schemaMap.get(ns);
572 s.setTargetNamespace(ns);
573 schemaMap.put(ns, s);
574 elementMap.put(s, new HashMap<String,Element>());
575 attributeMap.put(s, new HashMap<String, org.w3._2001.xmlschema.Attribute>());