1 package org.simantics.district.imports;
3 import java.io.IOException;
5 import java.nio.charset.Charset;
6 import java.nio.file.Files;
7 import java.nio.file.Path;
8 import java.util.ArrayList;
9 import java.util.Collection;
10 import java.util.HashMap;
11 import java.util.HashSet;
12 import java.util.Iterator;
13 import java.util.List;
15 import java.util.Optional;
17 import java.util.concurrent.atomic.AtomicInteger;
18 import java.util.function.Function;
20 import org.apache.commons.csv.CSVFormat;
21 import org.apache.commons.csv.CSVParser;
22 import org.apache.commons.csv.CSVRecord;
23 import org.geotools.geometry.DirectPosition2D;
24 import org.geotools.referencing.CRS;
25 import org.opengis.geometry.DirectPosition;
26 import org.opengis.geometry.MismatchedDimensionException;
27 import org.opengis.referencing.FactoryException;
28 import org.opengis.referencing.NoSuchAuthorityCodeException;
29 import org.opengis.referencing.crs.CoordinateReferenceSystem;
30 import org.opengis.referencing.operation.MathTransform;
31 import org.opengis.referencing.operation.TransformException;
32 import org.simantics.Simantics;
33 import org.simantics.databoard.Bindings;
34 import org.simantics.db.Resource;
35 import org.simantics.db.WriteGraph;
36 import org.simantics.db.common.request.WriteRequest;
37 import org.simantics.db.exception.DatabaseException;
38 import org.simantics.db.layer0.util.Layer0Utils;
39 import org.simantics.district.network.DNEdgeBuilder;
40 import org.simantics.district.network.DistrictNetworkUtil;
41 import org.simantics.district.network.ontology.DistrictNetworkResource;
42 import org.slf4j.Logger;
43 import org.slf4j.LoggerFactory;
45 import com.vividsolutions.jts.index.quadtree.Quadtree;
47 public class DistrictImportUtils {
49 private DistrictImportUtils() { }
51 private static final Logger LOGGER = LoggerFactory.getLogger(DistrictImportUtils.class);
53 public static Resource importCSVAsLayer(Path csvFile) throws IOException {
55 try (CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(Files.newBufferedReader(csvFile))) {
56 Map<String, Integer> header = parser.getHeaderMap();
57 System.out.println(header);
62 public static Map<String, Integer> readCSVHeader(Path source, char delimiter, boolean firstAsHeader) throws IOException {
63 return readCSVHeader(source, CSVFormat.newFormat(delimiter), firstAsHeader);
66 public static Map<String, Integer> readCSVHeader(Path source, CSVFormat format, boolean firstAsHeader) throws IOException {
68 format = format.withFirstRecordAsHeader();
69 try (CSVParser parser = format.parse(Files.newBufferedReader(source, Charset.defaultCharset()))) {
70 return parser.getHeaderMap();
74 public static Map<String, Character> getSupportedCSVDelimiterFormats() {
75 Map<String, Character> delimiters = new HashMap<>();
76 delimiters.put("Comma", ',');
77 delimiters.put("Semicolon", ';');
78 delimiters.put("Tabulator", '\t');
82 public static List<Map<String, String>> readRows(Path source, CSVFormat format, boolean firstAsHeader, int amount) throws IOException {
84 format = format.withFirstRecordAsHeader();
85 try (CSVParser parser = format.parse(Files.newBufferedReader(source))) {
87 List<Map<String, String>> results = new ArrayList<>(amount);
88 Iterator<CSVRecord> iter = parser.iterator();
89 while (start < amount && iter.hasNext()) {
90 CSVRecord record = iter.next();
91 results.add(record.toMap());
98 public static List<CSVRecord> readRows(Path source, char delim, boolean firstAsHeader, int rowAmount) throws IOException {
99 List<CSVRecord> results = new ArrayList<>();
100 AtomicInteger count = new AtomicInteger(0);
101 consumeCSV(source, delim, firstAsHeader, t -> {
103 int current = count.incrementAndGet();
104 return current < rowAmount;
109 public static void consumeCSV(Path source, char delim, boolean firstAsHeader, Function<CSVRecord, Boolean> consumer) throws IOException {
110 consumeCSV(Files.newBufferedReader(source), delim, firstAsHeader, consumer);
113 public static void consumeCSV(Reader reader, char delim, boolean firstAsHeader,
114 Function<CSVRecord, Boolean> consumer) throws IOException {
115 CSVFormat format = CSVFormat.newFormat(delim).withQuote('"');
117 format = format.withFirstRecordAsHeader();
119 try (CSVParser parser = format.parse(reader)) {
120 Iterator<CSVRecord> records = parser.iterator();
121 while (records.hasNext()) {
122 Boolean cont = consumer.apply(records.next());
131 public static Map<CSVHeader, List<String>> readCSVHeaderAndRows(Path source, char delimiter, boolean firstAsHeader, int amount) throws IOException {
132 Map<CSVHeader, List<String>> results = new HashMap<>();
133 CSVFormat format = CSVFormat.newFormat(delimiter);
135 format = format.withFirstRecordAsHeader();
136 try (CSVParser parser = format.parse(Files.newBufferedReader(source))) {
137 Map<String, Integer> headers = parser.getHeaderMap();
138 if (headers != null && !headers.isEmpty()) {
139 for (int index = 0; index < headers.size(); index++) {
140 for (String head : headers.keySet()) {
141 results.put(new CSVHeader(head, index), new ArrayList<>());
146 Iterator<CSVRecord> records = parser.iterator();
148 while (rows < amount && records.hasNext()) {
149 CSVRecord record = records.next();
150 for (int j = 0; j < record.size(); j++) {
151 String value = record.get(j);
152 String header = Integer.toString(j);
153 CSVHeader csvHeader = new CSVHeader(header, j);
154 List<String> vals = results.get(csvHeader);
156 vals = new ArrayList<>();
157 results.put(csvHeader, vals);
167 public static class CSVHeader {
169 private final String header;
170 private final int index;
172 public CSVHeader(String header, int index) {
173 this.header = header;
177 public String getHeader() {
181 public int getIndex() {
186 public int hashCode() {
187 final int prime = 31;
189 result = prime * result + ((header == null) ? 0 : header.hashCode());
190 result = prime * result + index;
195 public boolean equals(Object obj) {
200 if (getClass() != obj.getClass())
202 CSVHeader other = (CSVHeader) obj;
203 if (header == null) {
204 if (other.header != null)
206 } else if (!header.equals(other.header))
208 if (index != other.index)
214 public static Collection<String> readDistinctValuesOfColumn(Path source, char delim, int mappingIndex) throws IOException {
215 Set<String> results = new HashSet<>();
216 CSVFormat format = CSVFormat.newFormat(delim);
217 try (CSVParser parser = format.parse(Files.newBufferedReader(source))) {
218 Iterator<CSVRecord> records = parser.iterator();
219 if (records.hasNext())
221 while (records.hasNext()) {
222 CSVRecord row = records.next();
223 String value = row.get(mappingIndex);
230 public static void importVertices(CSVImportModel model) throws NoSuchAuthorityCodeException, FactoryException, DatabaseException {
232 Path csvFile = model.getSource();
233 char delim = model.getDelimiter();
235 int xCoordColumnIndex = model.getXCoordIndex();
236 int yCoordColumnIndex = model.getYCoordIndex();
237 int zCoordColumnIndex = model.getZCoordIndex();
238 int altElevationIndex = model.getAlternativeElevationIndex();
239 int supplyTempColumnIndex = model.getSupplyTempIndex();
240 int returnTempColumnIndex = model.getReturnTempIndex();
241 int supplyPressureColumnIndex = model.getSupplyPressureIndex();
242 int returnPressureColumnIndex = model.getReturnPressureIndex();
243 int dpIndex = model.getDeltaPressureIndex();
244 int dtIndex = model.getDeltaTemperatureIndex();
245 int heatPowerIndex = model.getHeatPowerIndex();
246 int peakPowerIndex = model.getPeakPowerIndex();
247 int valvePositionIndex = model.getValvePositionIndx();
248 int nominalHeadMIndex = model.getNominalHeadMIndex();
249 int nominalHeadBIndex = model.getNominalHeadBIndex();
250 int nominalFlowIndex = model.getNominalFlowIndex();
251 int maximumHeadMIndex = model.getMaximumHeadMIndex();
252 int heatLoadDsIndex = model.getHeatLoadDsIndex();
253 int massFlowIndex = model.getMassFlowIndex();
254 int volFlowIndex = model.getVolFlowIndex();
255 int velocityIndex = model.getVelocityIndex();
256 int flowAreaIndex = model.getFlowAreaIndex();
257 int nominalPressureLossIndex = model.getNominalPressureLossIndex();
258 int addressIndex = model.getAddressIndex();
259 int regionIndex = model.getRegionIndex();
261 int mappingColumn = model.getComponentMappingIndex();
262 int idColumn = model.getIdIndex();
264 String sourceEPSGCRS = model.getSourceCRS();
266 MathTransform transform = null;
267 boolean doTransform = false;
268 // if sourceEPSGCRS is empty || null then ignore transformation
269 if (sourceEPSGCRS != null && !sourceEPSGCRS.isEmpty()) {
270 CoordinateReferenceSystem sourceCRS = CRS.decode(sourceEPSGCRS);
271 CoordinateReferenceSystem targetCRS = CRS.decode("EPSG:4326");
272 transform = CRS.findMathTransform(sourceCRS, targetCRS, true);
275 final boolean actualDoTransform = doTransform;
276 final MathTransform actualTransform = transform;
278 Simantics.getSession().syncRequest(new WriteRequest() {
281 public void perform(WriteGraph graph) throws DatabaseException {
283 Layer0Utils.setDependenciesIndexingDisabled(graph, true);
284 graph.markUndoPoint();
286 DistrictNetworkResource DN = DistrictNetworkResource.getInstance(graph);
288 DistrictImportUtils.consumeCSV(csvFile, delim, true, (Function<CSVRecord, Boolean>) row -> {
290 String mappingValue = row.get(mappingColumn);
292 String xCoords = row.get(xCoordColumnIndex);
293 String yCoords = row.get(yCoordColumnIndex);
294 double xCoord = Double.parseDouble(xCoords);
295 double yCoord = Double.parseDouble(yCoords);
298 if (zCoordColumnIndex != -1) {
299 String zs = row.get(zCoordColumnIndex);
303 z = Double.parseDouble(zs);
304 } catch (NumberFormatException e1) {
305 throw new DatabaseException(e1);
311 if (actualDoTransform) {
312 DirectPosition2D targetPos = new DirectPosition2D();
313 DirectPosition2D sourcePos = new DirectPosition2D(xCoord, yCoord);
314 DirectPosition res = actualTransform.transform(sourcePos, targetPos);
315 coords = res.getCoordinate();
317 coords = new double[] { xCoord, yCoord };
320 // Switch to (longitude, latitude)
323 Resource vertex = DistrictNetworkUtil.createVertex(graph, model.getParentDiagram(), coords, z, model.getComponentMappings().get(mappingValue));
325 writeStringValue(graph, row, idColumn, vertex, DN.HasId);
327 writeValue(graph, row, altElevationIndex, vertex, DN.Vertex_HasAltElevation);
329 writeValue(graph, row, supplyTempColumnIndex, vertex, DN.Vertex_HasSupplyTemperature);
330 writeValue(graph, row, returnTempColumnIndex, vertex, DN.Vertex_HasReturnTemperature);
331 writeValue(graph, row, supplyPressureColumnIndex, vertex, DN.Vertex_HasSupplyPressure);
332 writeValue(graph, row, returnPressureColumnIndex, vertex, DN.Vertex_HasReturnPressure);
333 writeValue(graph, row, dpIndex, vertex, DN.Vertex_HasDeltaPressure);
334 writeValue(graph, row, dtIndex, vertex, DN.Vertex_HasDeltaTemperature);
335 writeValue(graph, row, heatPowerIndex, vertex, DN.Vertex_HasHeatPower);
336 writeValue(graph, row, peakPowerIndex, vertex, DN.Vertex_HasPeakPower);
337 writeValue(graph, row, valvePositionIndex, vertex, DN.Vertex_HasValvePosition);
338 writeValue(graph, row, nominalHeadMIndex, vertex, DN.Vertex_HasNominalHeadM);
339 writeValue(graph, row, nominalHeadBIndex, vertex, DN.Vertex_HasNominalHeadB);
340 writeValue(graph, row, nominalFlowIndex, vertex, DN.Vertex_HasNominalFlow);
341 writeValue(graph, row, maximumHeadMIndex, vertex, DN.Vertex_HasMaximumHeadM);
342 writeValue(graph, row, heatLoadDsIndex, vertex, DN.Vertex_HasHeatLoadDs);
343 writeValue(graph, row, massFlowIndex, vertex, DN.Vertex_HasMassFlow);
344 writeValue(graph, row, volFlowIndex, vertex, DN.Vertex_HasVolFlow);
345 writeValue(graph, row, velocityIndex, vertex, DN.Vertex_HasVelocity);
346 writeValue(graph, row, flowAreaIndex, vertex, DN.Vertex_HasFlowArea);
347 writeValue(graph, row, nominalPressureLossIndex, vertex, DN.Vertex_HasNominalPressureLoss);
348 writeStringValue(graph, row, addressIndex, vertex, DN.Vertex_HasAddress);
349 writeStringValue(graph, row, regionIndex, vertex, DN.HasRegion);
350 } catch (DatabaseException | MismatchedDimensionException | TransformException e) {
351 throw new RuntimeException(e);
356 } catch (IOException e) {
357 LOGGER.error("Could not import", e);
358 throw new DatabaseException(e);
360 Layer0Utils.setDependenciesIndexingDisabled(graph, false);
366 public static void importEdges(CSVImportModel model) throws NoSuchAuthorityCodeException, FactoryException, DatabaseException {
368 Path csvFile = model.getSource();
369 char delim = model.getDelimiter();
371 Set<String> writtenIds = new HashSet<>();
373 int startXCoordColumnIndex = model.getStartXCoordIndex();
374 int startYCoordColumnIndex = model.getStartYCoordIndex();
375 int startZValueColumnIndex = model.getStartZCoordIndex();
376 int endXCoordColumnIndex = model.getEndXCoordIndex();
377 int endYCoordColumnIndex = model.getEndYCoordIndex();
378 int endZValueColumnIndex = model.getEndZCoordIndex();
379 int diameterColumnIndex= model.getDiameterIndex();
380 int outerDiameterColumnIndex = model.getOuterDiamterIndex();
381 int nominalMassFlowIndex = model.getNominalMassFlowIndex();
382 int tGroundIndex = model.gettGroundIndex();
383 int edgeFlowAreaIndex = model.getEdgeFlowAreaIndex();
384 int kReturnIndex = model.getkReturnIndex();
385 int kSupplyIndex = model.getkSupplyIndex();
386 int lengthIndex = model.getLengthIndex();
387 int detailedGeometryIndex = model.getDetailedGeometryIndex();
388 int regionIndex = model.getRegionIndex();
389 int pipeTypeIndex = model.getPipeTypeIndex();
390 int pipeCodeIndex = model.getPipeCodeIndex();
391 int installationYearIndex = model.getInstallationYearIndex();
392 int wallThicknessIndex = model.getWallThicknessIndex();
393 int insulationConductivityIndex = model.getInsulationConductivityIndex();
394 int pipeSizeDNIndex = model.getPipeSizeDNIndex();
395 int roughnessIndex = model.getRoughnessIndex();
396 int structureIndex = model.getStructureIndex();
397 int conductanceIndex = model.getConductanceIndex();
399 int mappingColumn = model.getComponentMappingIndex();
400 int idColumn = model.getIdIndex();
402 double padding = model.getEdgePadding();
404 String sourceEPSGCRS = model.getSourceCRS();
406 MathTransform transform = null;
407 boolean doTransform = false;
408 // if sourceEPSGCRS is empty || null then ignore transformation
409 if (sourceEPSGCRS != null && !sourceEPSGCRS.isEmpty()) {
410 CoordinateReferenceSystem sourceCRS = CRS.decode(sourceEPSGCRS);
411 CoordinateReferenceSystem targetCRS = CRS.decode("EPSG:4326");
412 transform = CRS.findMathTransform(sourceCRS, targetCRS, true);
415 final boolean actualDoTransform = doTransform;
416 final MathTransform actualTransform = transform;
418 double halfPadding = padding / 2;
420 Quadtree existingVertices = DistrictNetworkUtil.existingVertices(model.getParentDiagram(), halfPadding);
422 Simantics.getSession().syncRequest(new WriteRequest() {
425 public void perform(WriteGraph graph) throws DatabaseException {
427 Layer0Utils.setDependenciesIndexingDisabled(graph, true);
428 graph.markUndoPoint();
430 DistrictNetworkResource DN = DistrictNetworkResource.getInstance(graph);
432 DistrictImportUtils.consumeCSV(csvFile, delim, true, row -> {
435 String idValue = row.get(idColumn);
436 if (!writtenIds.contains(idValue)) {
437 writtenIds.add(idValue);
438 String mappingValue = row.get(mappingColumn);
440 String startXCoords = row.get(startXCoordColumnIndex);
441 String startYCoords = row.get(startYCoordColumnIndex);
442 String startZCoords = row.get(startZValueColumnIndex);
443 String endXCoords = row.get(endXCoordColumnIndex);
444 String endYCoords = row.get(endYCoordColumnIndex);
445 String endZCoords = row.get(endZValueColumnIndex);
447 double startXCoord = Double.parseDouble(startXCoords); // make negative
448 double startYCoord = Double.parseDouble(startYCoords);
449 double startZCoord = Double.parseDouble(startZCoords);
451 double endXCoord = Double.parseDouble(endXCoords); // make negative
452 double endYCoord = Double.parseDouble(endYCoords);
453 double endZCoord = Double.parseDouble(endZCoords);
455 double[] startCoords;
457 if (actualDoTransform) {
458 DirectPosition2D startTargetPos = new DirectPosition2D();
459 DirectPosition2D startSourcePos = new DirectPosition2D(startXCoord, startYCoord);
460 DirectPosition startRes = actualTransform.transform(startSourcePos, startTargetPos);
461 startCoords = startRes.getCoordinate();
463 DirectPosition2D endTargetPos = new DirectPosition2D();
464 DirectPosition2D endSourcePos = new DirectPosition2D(endXCoord, endYCoord);
465 DirectPosition endRes = actualTransform.transform(endSourcePos, endTargetPos);
466 endCoords = endRes.getCoordinate();
468 startCoords = new double[] { startXCoord , startYCoord };
469 endCoords = new double[] { endXCoord , endYCoord };
472 // Switch to (longitude, latitude)
473 flipAxes(startCoords);
476 Optional<Resource> oedge = DNEdgeBuilder.create(graph, existingVertices, model.getParentDiagram(), model.getComponentMappings().get(mappingValue), startCoords, startZCoord, endCoords, endZCoord, new double[0], padding, true);
477 if (oedge.isPresent()) {
478 Resource edge = oedge.get();
480 writeStringValue(graph, row, idColumn, edge, DN.HasId);
482 writeValue(graph, row, diameterColumnIndex, edge, DN.Edge_HasDiameter);
483 writeValue(graph, row, outerDiameterColumnIndex, edge, DN.Edge_HasOuterDiameter);
484 writeValue(graph, row, nominalMassFlowIndex, edge, DN.Edge_HasNominalMassFlow);
485 writeValue(graph, row, tGroundIndex, edge, DN.Edge_HasTGround);
486 writeValue(graph, row, kReturnIndex, edge, DN.Edge_HasKReturn);
487 writeValue(graph, row, kSupplyIndex, edge, DN.Edge_HasKSupply);
488 writeValue(graph, row, edgeFlowAreaIndex, edge, DN.Edge_HasFlowArea);
489 writeValue(graph, row, lengthIndex, edge, DN.Edge_HasLength);
490 writeStringValue(graph, row, regionIndex, edge, DN.HasRegion);
491 writeStringValue(graph, row, pipeTypeIndex, edge, DN.Edge_HasType);
492 writeDoubleArrayFromString(graph, row, detailedGeometryIndex, edge, DN.Edge_HasGeometry, actualTransform);
493 writeStringValue(graph, row, pipeCodeIndex, edge, DN.Edge_HasPipeCode);
494 writeIntegerValue(graph, row, installationYearIndex, edge, DN.Edge_HasInstallationYear);
495 writeValue(graph, row, wallThicknessIndex, edge, DN.Edge_HasWallThickness);
496 writeValue(graph, row, insulationConductivityIndex, edge, DN.Edge_HasInsulationConductivity);
497 writeIntegerValue(graph, row, pipeSizeDNIndex, edge, DN.Edge_HasPipeSizeDN);
498 writeValue(graph, row, roughnessIndex, edge, DN.Edge_HasRoughness);
499 writeStringValue(graph, row, structureIndex, edge, DN.Edge_HasStructure);
500 writeValue(graph, row, conductanceIndex, edge, DN.Edge_HasConductance);
504 } catch (DatabaseException | MismatchedDimensionException | TransformException e) {
505 throw new RuntimeException(e);
508 } catch (IOException e) {
509 LOGGER.error("Could not import edges {}", model.getSource(), e);
511 Layer0Utils.setDependenciesIndexingDisabled(graph, false);
517 private static void flipAxes(double[] coords) {
518 double tmp = coords[0];
519 coords[0] = coords[1];
523 private static void writeValue(WriteGraph graph, CSVRecord row, int index, Resource subject, Resource relation) throws DatabaseException {
525 String stringValue = row.get(index);
526 if (!stringValue.isEmpty()) {
528 if (stringValue.startsWith("\"") && stringValue.endsWith("\"")) {
529 stringValue = stringValue.substring(1, stringValue.length() - 1);
531 graph.claimLiteral(subject, relation, Double.parseDouble(stringValue), Bindings.DOUBLE);
532 } catch (NumberFormatException e) {
533 LOGGER.error("Could not parse {} {} {} {}", row, index, subject, relation, e);
534 //throw new DatabaseException(e);
540 private static void writeIntegerValue(WriteGraph graph, CSVRecord row, int index, Resource subject, Resource relation) throws DatabaseException {
542 String stringValue = row.get(index);
543 if (!stringValue.isEmpty()) {
545 if (stringValue.startsWith("\"") && stringValue.endsWith("\"")) {
546 stringValue = stringValue.substring(1, stringValue.length() - 1);
548 graph.claimLiteral(subject, relation, Integer.parseInt(stringValue), Bindings.INTEGER);
549 } catch (NumberFormatException e) {
550 LOGGER.error("Could not parse {} {} {} {}", row, index, subject, relation, e);
551 //throw new DatabaseException(e);
557 private static void writeStringValue(WriteGraph graph, CSVRecord row, int index, Resource subject, Resource relation) throws DatabaseException {
559 String stringValue = row.get(index);
560 if (!stringValue.isEmpty()) {
562 graph.claimLiteral(subject, relation, stringValue, Bindings.STRING);
563 } catch (NumberFormatException e) {
564 throw new DatabaseException(e);
570 private static void writeDoubleArrayFromString(WriteGraph graph, CSVRecord row, int index, Resource subject, Resource relation, MathTransform actualTransform) throws DatabaseException, MismatchedDimensionException, TransformException {
572 String stringValue = row.get(index);
573 if (!stringValue.isEmpty()) {
574 if (stringValue.startsWith("\"") && stringValue.endsWith("\"")) {
575 stringValue = stringValue.substring(1, stringValue.length() - 1);
577 String[] coordPairs = stringValue.split(";");
578 ArrayList<Double> dd = new ArrayList<>(coordPairs.length * 2);
579 for (int i = 0; i < coordPairs.length; i++) {
580 String coordPair = coordPairs[i];
581 String[] p = coordPair.split(" ");
582 double x = Double.parseDouble(p[0]);
583 double y = Double.parseDouble(p[1]);
584 if (actualTransform != null) {
585 DirectPosition2D targetPos = new DirectPosition2D();
586 DirectPosition2D sourcePos = new DirectPosition2D(y, x);
587 DirectPosition res = actualTransform.transform(sourcePos, targetPos);
588 double[] coords = res.getCoordinate();
595 double[] detailedGeometryCoords = new double[dd.size()];
596 for (int i = 0; i < dd.size(); i++) {
597 double d = dd.get(i);
598 detailedGeometryCoords[i] = d;
601 graph.claimLiteral(subject, relation, detailedGeometryCoords, Bindings.DOUBLE_ARRAY);
602 } catch (NumberFormatException e) {
603 throw new DatabaseException(e);