1 package org.simantics.district.imports;
3 import java.io.IOException;
4 import java.nio.file.Files;
5 import java.nio.file.Path;
6 import java.util.ArrayList;
7 import java.util.Collection;
8 import java.util.HashMap;
9 import java.util.HashSet;
10 import java.util.Iterator;
11 import java.util.List;
13 import java.util.Optional;
15 import java.util.concurrent.atomic.AtomicInteger;
16 import java.util.function.Function;
18 import org.apache.commons.csv.CSVFormat;
19 import org.apache.commons.csv.CSVParser;
20 import org.apache.commons.csv.CSVRecord;
21 import org.geotools.geometry.DirectPosition2D;
22 import org.geotools.referencing.CRS;
23 import org.opengis.geometry.DirectPosition;
24 import org.opengis.geometry.MismatchedDimensionException;
25 import org.opengis.referencing.FactoryException;
26 import org.opengis.referencing.NoSuchAuthorityCodeException;
27 import org.opengis.referencing.crs.CoordinateReferenceSystem;
28 import org.opengis.referencing.operation.MathTransform;
29 import org.opengis.referencing.operation.TransformException;
30 import org.simantics.Simantics;
31 import org.simantics.databoard.Bindings;
32 import org.simantics.db.Resource;
33 import org.simantics.db.WriteGraph;
34 import org.simantics.db.common.request.WriteRequest;
35 import org.simantics.db.exception.DatabaseException;
36 import org.simantics.db.layer0.util.Layer0Utils;
37 import org.simantics.district.network.DNEdgeBuilder;
38 import org.simantics.district.network.DistrictNetworkUtil;
39 import org.simantics.district.network.ontology.DistrictNetworkResource;
40 import org.slf4j.Logger;
41 import org.slf4j.LoggerFactory;
43 import com.vividsolutions.jts.index.quadtree.Quadtree;
45 public class DistrictImportUtils {
47 private DistrictImportUtils() { }
49 private static final Logger LOGGER = LoggerFactory.getLogger(DistrictImportUtils.class);
51 public static Resource importCSVAsLayer(Path csvFile) throws IOException {
53 try (CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(Files.newBufferedReader(csvFile))) {
54 Map<String, Integer> header = parser.getHeaderMap();
55 System.out.println(header);
60 public static Map<String, Integer> readCSVHeader(Path source, char delimiter, boolean firstAsHeader) throws IOException {
61 return readCSVHeader(source, CSVFormat.newFormat(delimiter), firstAsHeader);
64 public static Map<String, Integer> readCSVHeader(Path source, CSVFormat format, boolean firstAsHeader) throws IOException {
66 format = format.withFirstRecordAsHeader();
67 try (CSVParser parser = format.parse(Files.newBufferedReader(source))) {
68 return parser.getHeaderMap();
72 public static Map<String, Character> getSupportedCSVDelimiterFormats() {
73 Map<String, Character> delimiters = new HashMap<>();
74 delimiters.put("Comma", ',');
75 delimiters.put("Semicolon", ';');
76 delimiters.put("Tabulator", '\t');
80 public static List<Map<String, String>> readRows(Path source, CSVFormat format, boolean firstAsHeader, int amount) throws IOException {
82 format = format.withFirstRecordAsHeader();
83 try (CSVParser parser = format.parse(Files.newBufferedReader(source))) {
85 List<Map<String, String>> results = new ArrayList<>(amount);
86 Iterator<CSVRecord> iter = parser.iterator();
87 while (start < amount && iter.hasNext()) {
88 CSVRecord record = iter.next();
89 results.add(record.toMap());
96 public static List<CSVRecord> readRows(Path source, char delim, boolean firstAsHeader, int rowAmount) throws IOException {
97 List<CSVRecord> results = new ArrayList<>();
98 AtomicInteger count = new AtomicInteger(0);
99 consumeCSV(source, delim, firstAsHeader, t -> {
101 int current = count.incrementAndGet();
102 return current < rowAmount;
107 public static void consumeCSV(Path source, char delim, boolean firstAsHeader, Function<CSVRecord, Boolean> consumer) throws IOException {
108 CSVFormat format = CSVFormat.newFormat(delim);
110 format = format.withFirstRecordAsHeader();
112 try (CSVParser parser = format.parse(Files.newBufferedReader(source))) {
113 Iterator<CSVRecord> records = parser.iterator();
114 while (records.hasNext()) {
115 Boolean cont = consumer.apply(records.next());
124 public static Map<CSVHeader, List<String>> readCSVHeaderAndRows(Path source, char delimiter, boolean firstAsHeader, int amount) throws IOException {
125 Map<CSVHeader, List<String>> results = new HashMap<>();
126 CSVFormat format = CSVFormat.newFormat(delimiter);
128 format = format.withFirstRecordAsHeader();
129 try (CSVParser parser = format.parse(Files.newBufferedReader(source))) {
130 Map<String, Integer> headers = parser.getHeaderMap();
131 if (headers != null && !headers.isEmpty()) {
132 for (int index = 0; index < headers.size(); index++) {
133 for (String head : headers.keySet()) {
134 results.put(new CSVHeader(head, index), new ArrayList<>());
139 Iterator<CSVRecord> records = parser.iterator();
141 while (rows < amount && records.hasNext()) {
142 CSVRecord record = records.next();
143 for (int j = 0; j < record.size(); j++) {
144 String value = record.get(j);
145 String header = Integer.toString(j);
146 CSVHeader csvHeader = new CSVHeader(header, j);
147 List<String> vals = results.get(csvHeader);
149 vals = new ArrayList<>();
150 results.put(csvHeader, vals);
160 public static class CSVHeader {
162 private final String header;
163 private final int index;
165 public CSVHeader(String header, int index) {
166 this.header = header;
170 public String getHeader() {
174 public int getIndex() {
179 public int hashCode() {
180 final int prime = 31;
182 result = prime * result + ((header == null) ? 0 : header.hashCode());
183 result = prime * result + index;
188 public boolean equals(Object obj) {
193 if (getClass() != obj.getClass())
195 CSVHeader other = (CSVHeader) obj;
196 if (header == null) {
197 if (other.header != null)
199 } else if (!header.equals(other.header))
201 if (index != other.index)
207 public static Collection<String> readDistinctValuesOfColumn(Path source, char delim, int mappingIndex) throws IOException {
208 Set<String> results = new HashSet<>();
209 CSVFormat format = CSVFormat.newFormat(delim);
210 try (CSVParser parser = format.parse(Files.newBufferedReader(source))) {
211 Iterator<CSVRecord> records = parser.iterator();
212 if (records.hasNext())
214 while (records.hasNext()) {
215 CSVRecord row = records.next();
216 String value = row.get(mappingIndex);
223 public static void importVertices(CSVImportModel model) throws NoSuchAuthorityCodeException, FactoryException, DatabaseException {
225 Path csvFile = model.getSource();
226 char delim = model.getDelimiter();
228 int xCoordColumnIndex = model.getXCoordIndex();
229 int yCoordColumnIndex = model.getYCoordIndex();
230 int zCoordColumnIndex = model.getZCoordIndex();
231 int altElevationIndex = model.getAlternativeElevationIndex();
232 int supplyTempColumnIndex = model.getSupplyTempIndex();
233 int returnTempColumnIndex = model.getReturnTempIndex();
234 int supplyPressureColumnIndex = model.getSupplyPressureIndex();
235 int returnPressureColumnIndex = model.getReturnPressureIndex();
236 int dpIndex = model.getDeltaPressureIndex();
237 int dtIndex = model.getDeltaTemperatureIndex();
238 int heatPowerIndex = model.getHeatPowerIndex();
239 int peakPowerIndex = model.getPeakPowerIndex();
240 int valvePositionIndex = model.getValvePositionIndx();
241 int nominalHeadMIndex = model.getNominalHeadMIndex();
242 int nominalHeadBIndex = model.getNominalHeadBIndex();
243 int nominalFlowIndex = model.getNominalFlowIndex();
244 int maximumHeadMIndex = model.getMaximumHeadMIndex();
245 int heatLoadDsIndex = model.getHeatLoadDsIndex();
246 int massFlowIndex = model.getMassFlowIndex();
247 int volFlowIndex = model.getVolFlowIndex();
248 int velocityIndex = model.getVelocityIndex();
249 int flowAreaIndex = model.getFlowAreaIndex();
250 int nominalPressureLossIndex = model.getNominalPressureLossIndex();
251 int addressIndex = model.getAddressIndex();
252 int regionIndex = model.getRegionIndex();
254 int mappingColumn = model.getComponentMappingIndex();
255 int idColumn = model.getIdIndex();
257 String sourceEPSGCRS = model.getSourceCRS();
259 MathTransform transform = null;
260 boolean doTransform = false;
261 // if sourceEPSGCRS is empty || null then ignore transformation
262 if (sourceEPSGCRS != null && !sourceEPSGCRS.isEmpty()) {
263 CoordinateReferenceSystem sourceCRS = CRS.decode(sourceEPSGCRS);
264 CoordinateReferenceSystem targetCRS = CRS.decode("EPSG:4326");
265 transform = CRS.findMathTransform(sourceCRS, targetCRS, true);
268 final boolean actualDoTransform = doTransform;
269 final MathTransform actualTransform = transform;
271 Simantics.getSession().syncRequest(new WriteRequest() {
274 public void perform(WriteGraph graph) throws DatabaseException {
276 Layer0Utils.setDependenciesIndexingDisabled(graph, true);
277 graph.markUndoPoint();
279 DistrictNetworkResource DN = DistrictNetworkResource.getInstance(graph);
281 DistrictImportUtils.consumeCSV(csvFile, delim, true, (Function<CSVRecord, Boolean>) row -> {
283 String mappingValue = row.get(mappingColumn);
285 String xCoords = row.get(xCoordColumnIndex);
286 String yCoords = row.get(yCoordColumnIndex);
287 double xCoord = Double.parseDouble(xCoords);
288 double yCoord = Double.parseDouble(yCoords);
291 if (zCoordColumnIndex != -1) {
292 String zs = row.get(zCoordColumnIndex);
296 z = Double.parseDouble(zs);
297 } catch (NumberFormatException e1) {
298 throw new DatabaseException(e1);
304 if (actualDoTransform) {
305 DirectPosition2D targetPos = new DirectPosition2D();
306 DirectPosition2D sourcePos = new DirectPosition2D(xCoord, yCoord);
307 DirectPosition res = actualTransform.transform(sourcePos, targetPos);
308 coords = res.getCoordinate();
310 coords = new double[] { xCoord, yCoord };
313 // Switch to (longitude, latitude)
316 Resource vertex = DistrictNetworkUtil.createVertex(graph, model.getParentDiagram(), coords, z, model.getComponentMappings().get(mappingValue));
318 writeStringValue(graph, row, idColumn, vertex, DN.HasId);
320 writeValue(graph, row, altElevationIndex, vertex, DN.Vertex_HasAltElevation);
322 writeValue(graph, row, supplyTempColumnIndex, vertex, DN.Vertex_HasSupplyTemperature);
323 writeValue(graph, row, returnTempColumnIndex, vertex, DN.Vertex_HasReturnTemperature);
324 writeValue(graph, row, supplyPressureColumnIndex, vertex, DN.Vertex_HasSupplyPressure);
325 writeValue(graph, row, returnPressureColumnIndex, vertex, DN.Vertex_HasReturnPressure);
326 writeValue(graph, row, dpIndex, vertex, DN.Vertex_HasDeltaPressure);
327 writeValue(graph, row, dtIndex, vertex, DN.Vertex_HasDeltaTemperature);
328 writeValue(graph, row, heatPowerIndex, vertex, DN.Vertex_HasHeatPower);
329 writeValue(graph, row, peakPowerIndex, vertex, DN.Vertex_HasPeakPower);
330 writeValue(graph, row, valvePositionIndex, vertex, DN.Vertex_HasValvePosition);
331 writeValue(graph, row, nominalHeadMIndex, vertex, DN.Vertex_HasNominalHeadM);
332 writeValue(graph, row, nominalHeadBIndex, vertex, DN.Vertex_HasNominalHeadB);
333 writeValue(graph, row, nominalFlowIndex, vertex, DN.Vertex_HasNominalFlow);
334 writeValue(graph, row, maximumHeadMIndex, vertex, DN.Vertex_HasMaximumHeadM);
335 writeValue(graph, row, heatLoadDsIndex, vertex, DN.Vertex_HasHeatLoadDs);
336 writeValue(graph, row, massFlowIndex, vertex, DN.Vertex_HasMassFlow);
337 writeValue(graph, row, volFlowIndex, vertex, DN.Vertex_HasVolFlow);
338 writeValue(graph, row, velocityIndex, vertex, DN.Vertex_HasVelocity);
339 writeValue(graph, row, flowAreaIndex, vertex, DN.Vertex_HasFlowArea);
340 writeValue(graph, row, nominalPressureLossIndex, vertex, DN.Vertex_HasNominalPressureLoss);
341 writeStringValue(graph, row, addressIndex, vertex, DN.Vertex_HasAddress);
342 writeStringValue(graph, row, regionIndex, vertex, DN.HasRegion);
343 } catch (DatabaseException | MismatchedDimensionException | TransformException e) {
344 throw new RuntimeException(e);
349 } catch (IOException e) {
350 LOGGER.error("Could not import", e);
351 throw new DatabaseException(e);
353 Layer0Utils.setDependenciesIndexingDisabled(graph, false);
359 public static void importEdges(CSVImportModel model) throws NoSuchAuthorityCodeException, FactoryException, DatabaseException {
361 Path csvFile = model.getSource();
362 char delim = model.getDelimiter();
364 Set<String> writtenIds = new HashSet<>();
366 int startXCoordColumnIndex = model.getStartXCoordIndex();
367 int startYCoordColumnIndex = model.getStartYCoordIndex();
368 int startZValueColumnIndex = model.getStartZCoordIndex();
369 int endXCoordColumnIndex = model.getEndXCoordIndex();
370 int endYCoordColumnIndex = model.getEndYCoordIndex();
371 int endZValueColumnIndex = model.getEndZCoordIndex();
372 int diameterColumnIndex= model.getDiameterIndex();
373 int outerDiameterColumnIndex = model.getOuterDiamterIndex();
374 int nominalMassFlowIndex = model.getNominalMassFlowIndex();
375 int tGroundIndex = model.gettGroundIndex();
376 int edgeFlowAreaIndex = model.getEdgeFlowAreaIndex();
377 int kReturnIndex = model.getkReturnIndex();
378 int kSupplyIndex = model.getkSupplyIndex();
379 int lengthIndex = model.getLengthIndex();
380 int detailedGeometryIndex = model.getDetailedGeometryIndex();
381 int regionIndex = model.getRegionIndex();
382 int pipeTypeIndex = model.getPipeTypeIndex();
384 int mappingColumn = model.getComponentMappingIndex();
385 int idColumn = model.getIdIndex();
387 double padding = model.getEdgePadding();
389 String sourceEPSGCRS = model.getSourceCRS();
391 MathTransform transform = null;
392 boolean doTransform = false;
393 // if sourceEPSGCRS is empty || null then ignore transformation
394 if (sourceEPSGCRS != null && !sourceEPSGCRS.isEmpty()) {
395 CoordinateReferenceSystem sourceCRS = CRS.decode(sourceEPSGCRS);
396 CoordinateReferenceSystem targetCRS = CRS.decode("EPSG:4326");
397 transform = CRS.findMathTransform(sourceCRS, targetCRS, true);
400 final boolean actualDoTransform = doTransform;
401 final MathTransform actualTransform = transform;
403 double halfPadding = padding / 2;
405 Quadtree existingVertices = DistrictNetworkUtil.existingVertices(model.getParentDiagram(), halfPadding);
407 Simantics.getSession().syncRequest(new WriteRequest() {
410 public void perform(WriteGraph graph) throws DatabaseException {
412 Layer0Utils.setDependenciesIndexingDisabled(graph, true);
413 graph.markUndoPoint();
415 DistrictNetworkResource DN = DistrictNetworkResource.getInstance(graph);
417 DistrictImportUtils.consumeCSV(csvFile, delim, true, row -> {
420 String idValue = row.get(idColumn);
421 if (!writtenIds.contains(idValue)) {
422 writtenIds.add(idValue);
423 String mappingValue = row.get(mappingColumn);
425 String startXCoords = row.get(startXCoordColumnIndex);
426 String startYCoords = row.get(startYCoordColumnIndex);
427 String startZCoords = row.get(startZValueColumnIndex);
428 String endXCoords = row.get(endXCoordColumnIndex);
429 String endYCoords = row.get(endYCoordColumnIndex);
430 String endZCoords = row.get(endZValueColumnIndex);
432 double startXCoord = Double.parseDouble(startXCoords); // make negative
433 double startYCoord = Double.parseDouble(startYCoords);
434 double startZCoord = Double.parseDouble(startZCoords);
436 double endXCoord = Double.parseDouble(endXCoords); // make negative
437 double endYCoord = Double.parseDouble(endYCoords);
438 double endZCoord = Double.parseDouble(endZCoords);
440 double[] startCoords;
442 if (actualDoTransform) {
443 DirectPosition2D startTargetPos = new DirectPosition2D();
444 DirectPosition2D startSourcePos = new DirectPosition2D(startXCoord, startYCoord);
445 DirectPosition startRes = actualTransform.transform(startSourcePos, startTargetPos);
446 startCoords = startRes.getCoordinate();
448 DirectPosition2D endTargetPos = new DirectPosition2D();
449 DirectPosition2D endSourcePos = new DirectPosition2D(endXCoord, endYCoord);
450 DirectPosition endRes = actualTransform.transform(endSourcePos, endTargetPos);
451 endCoords = endRes.getCoordinate();
453 startCoords = new double[] { startXCoord , startYCoord };
454 endCoords = new double[] { endXCoord , endYCoord };
457 // Switch to (longitude, latitude)
458 flipAxes(startCoords);
461 Optional<Resource> oedge = DNEdgeBuilder.create(graph, existingVertices, model.getParentDiagram(), model.getComponentMappings().get(mappingValue), startCoords, startZCoord, endCoords, endZCoord, new double[0], padding, true);
462 if (oedge.isPresent()) {
463 Resource edge = oedge.get();
465 writeStringValue(graph, row, idColumn, edge, DN.HasId);
467 writeValue(graph, row, diameterColumnIndex, edge, DN.Edge_HasDiameter);
468 writeValue(graph, row, outerDiameterColumnIndex, edge, DN.Edge_HasOuterDiameter);
469 writeValue(graph, row, nominalMassFlowIndex, edge, DN.Edge_HasNominalMassFlow);
470 writeValue(graph, row, tGroundIndex, edge, DN.Edge_HasTGround);
471 writeValue(graph, row, kReturnIndex, edge, DN.Edge_HasKReturn);
472 writeValue(graph, row, kSupplyIndex, edge, DN.Edge_HasKSupply);
473 writeValue(graph, row, edgeFlowAreaIndex, edge, DN.Edge_HasFlowArea);
474 writeValue(graph, row, lengthIndex, edge, DN.Edge_HasLength);
475 writeStringValue(graph, row, regionIndex, edge, DN.HasRegion);
476 writeStringValue(graph, row, pipeTypeIndex, edge, DN.Edge_HasType);
477 writeDoubleArrayFromString(graph, row, detailedGeometryIndex, edge, DN.Edge_HasGeometry, actualTransform);
481 } catch (DatabaseException | MismatchedDimensionException | TransformException e) {
482 throw new RuntimeException(e);
485 } catch (IOException e) {
486 LOGGER.error("Could not import edges {}", model.getSource(), e);
488 Layer0Utils.setDependenciesIndexingDisabled(graph, false);
494 private static void flipAxes(double[] coords) {
495 double tmp = coords[0];
496 coords[0] = coords[1];
500 private static void writeValue(WriteGraph graph, CSVRecord row, int index, Resource subject, Resource relation) throws DatabaseException {
502 String stringValue = row.get(index);
503 if (!stringValue.isEmpty()) {
505 if (stringValue.startsWith("\"") && stringValue.endsWith("\"")) {
506 stringValue = stringValue.substring(1, stringValue.length() - 1);
508 graph.claimLiteral(subject, relation, Double.parseDouble(stringValue), Bindings.DOUBLE);
509 } catch (NumberFormatException e) {
510 LOGGER.error("Could not parse {} {} {} {}", row, index, subject, relation, e);
511 //throw new DatabaseException(e);
517 private static void writeStringValue(WriteGraph graph, CSVRecord row, int index, Resource subject, Resource relation) throws DatabaseException {
519 String stringValue = row.get(index);
520 if (!stringValue.isEmpty()) {
522 graph.claimLiteral(subject, relation, stringValue, Bindings.STRING);
523 } catch (NumberFormatException e) {
524 throw new DatabaseException(e);
530 private static void writeDoubleArrayFromString(WriteGraph graph, CSVRecord row, int index, Resource subject, Resource relation, MathTransform actualTransform) throws DatabaseException, MismatchedDimensionException, TransformException {
532 String stringValue = row.get(index);
533 if (!stringValue.isEmpty()) {
534 if (stringValue.startsWith("\"") && stringValue.endsWith("\"")) {
535 stringValue = stringValue.substring(1, stringValue.length() - 1);
537 String[] coordPairs = stringValue.split(";");
538 ArrayList<Double> dd = new ArrayList<>(coordPairs.length * 2);
539 for (int i = 0; i < coordPairs.length; i++) {
540 String coordPair = coordPairs[i];
541 String[] p = coordPair.split(" ");
542 double x = Double.parseDouble(p[0]);
543 double y = Double.parseDouble(p[1]);
544 if (actualTransform != null) {
545 DirectPosition2D targetPos = new DirectPosition2D();
546 DirectPosition2D sourcePos = new DirectPosition2D(y, x);
547 DirectPosition res = actualTransform.transform(sourcePos, targetPos);
548 double[] coords = res.getCoordinate();
555 double[] detailedGeometryCoords = new double[dd.size()];
556 for (int i = 0; i < dd.size(); i++) {
557 double d = dd.get(i);
558 detailedGeometryCoords[i] = d;
561 graph.claimLiteral(subject, relation, detailedGeometryCoords, Bindings.DOUBLE_ARRAY);
562 } catch (NumberFormatException e) {
563 throw new DatabaseException(e);