From: jsimomaa Date: Tue, 20 Dec 2016 15:25:07 +0000 (+0200) Subject: Adding basic import utilities - WIP! X-Git-Tag: v1.31.0~21 X-Git-Url: https://gerrit.simantics.org/r/gitweb?a=commitdiff_plain;h=31bcbe46c93a576cd6ade673a745ea9cb28672da;p=simantics%2Fdistrict.git Adding basic import utilities - WIP! --- diff --git a/org.simantics.district.feature/.project b/org.simantics.district.feature/.project new file mode 100644 index 00000000..61ffe362 --- /dev/null +++ b/org.simantics.district.feature/.project @@ -0,0 +1,17 @@ + + + org.simantics.district.feature + + + + + + org.eclipse.pde.FeatureBuilder + + + + + + org.eclipse.pde.FeatureNature + + diff --git a/org.simantics.district.feature/build.properties b/org.simantics.district.feature/build.properties new file mode 100644 index 00000000..82ab19c6 --- /dev/null +++ b/org.simantics.district.feature/build.properties @@ -0,0 +1 @@ +bin.includes = feature.xml diff --git a/org.simantics.district.feature/feature.xml b/org.simantics.district.feature/feature.xml new file mode 100644 index 00000000..57e342d4 --- /dev/null +++ b/org.simantics.district.feature/feature.xml @@ -0,0 +1,40 @@ + + + + + [Enter Feature Description here.] + + + + [Enter Copyright Description here.] + + + + [Enter License Description here.] + + + + + + + + + diff --git a/org.simantics.district.imports.ui/.classpath b/org.simantics.district.imports.ui/.classpath new file mode 100644 index 00000000..b862a296 --- /dev/null +++ b/org.simantics.district.imports.ui/.classpath @@ -0,0 +1,7 @@ + + + + + + + diff --git a/org.simantics.district.imports.ui/.project b/org.simantics.district.imports.ui/.project new file mode 100644 index 00000000..9c9d811b --- /dev/null +++ b/org.simantics.district.imports.ui/.project @@ -0,0 +1,28 @@ + + + org.simantics.district.imports.ui + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.pde.ManifestBuilder + + + + + org.eclipse.pde.SchemaBuilder + + + + + + org.eclipse.pde.PluginNature + org.eclipse.jdt.core.javanature + + diff --git a/org.simantics.district.imports.ui/META-INF/MANIFEST.MF b/org.simantics.district.imports.ui/META-INF/MANIFEST.MF new file mode 100644 index 00000000..fb65af54 --- /dev/null +++ b/org.simantics.district.imports.ui/META-INF/MANIFEST.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Simantics District Import UI +Bundle-SymbolicName: org.simantics.district.imports.ui;singleton:=true +Bundle-Version: 1.0.0.qualifier +Bundle-RequiredExecutionEnvironment: JavaSE-1.8 +Require-Bundle: org.eclipse.ui.ide, + org.eclipse.jface, + org.simantics.utils.ui, + org.simantics.district.imports;bundle-version="1.0.0", + org.simantics.db +Bundle-ActivationPolicy: lazy diff --git a/org.simantics.district.imports.ui/build.properties b/org.simantics.district.imports.ui/build.properties new file mode 100644 index 00000000..6f20375d --- /dev/null +++ b/org.simantics.district.imports.ui/build.properties @@ -0,0 +1,5 @@ +source.. = src/ +output.. = bin/ +bin.includes = META-INF/,\ + .,\ + plugin.xml diff --git a/org.simantics.district.imports.ui/plugin.xml b/org.simantics.district.imports.ui/plugin.xml new file mode 100644 index 00000000..fe77b996 --- /dev/null +++ b/org.simantics.district.imports.ui/plugin.xml @@ -0,0 +1,15 @@ + + + + + + + Imports CSV data to Simantics District network + + + + diff --git a/org.simantics.district.imports.ui/src/org/simantics/district/imports/ui/CSVImportModel.java b/org.simantics.district.imports.ui/src/org/simantics/district/imports/ui/CSVImportModel.java new file mode 100644 index 00000000..50c427c4 --- /dev/null +++ b/org.simantics.district.imports.ui/src/org/simantics/district/imports/ui/CSVImportModel.java @@ -0,0 +1,73 @@ +package org.simantics.district.imports.ui; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.simantics.district.imports.DistrictImportUtils; +import org.simantics.district.imports.DistrictImportUtils.CSVHeader; + +public class CSVImportModel { + + private Path source; + private char delimiter; + private boolean read; + private Map delimiters; + + public CSVImportModel() { + delimiters = DistrictImportUtils.getSupportedCSVDelimiterFormats(); + } + + public void setSource(Path source) { + this.source = source; + } + + public void setDelimiter(char delimiter) { + this.delimiter = delimiter; + } + + public void setReadFirstAsHeader(boolean read) { + this.read = read; + } + + public Path getSource() { + return source; + } + + public Map> getHeaderAndRows(int amount) throws IOException { + if (source != null) + return DistrictImportUtils.readCSVHeaderAndRows(source, delimiter, read, amount); + else + return Collections.emptyMap(); + } + + public Map getHeader() throws IOException { + Map header = null; + if (source != null) + header = DistrictImportUtils.readCSVHeader(source, delimiter, read); + if (header == null) + header = Collections.emptyMap(); + return header; + } + + public boolean getReadFirstAsHeader() { + return read; + } + + public String[] getDelimiterFormats() { + return delimiters.keySet().toArray(new String[delimiters.size()]); + } + + public void setDelimiterByLabel(String item) { + setDelimiter(delimiters.get(item)); + } + + public List> readRows(int amount) throws IOException { + if (source != null) + return DistrictImportUtils.readRows(source, delimiter, read, amount); + else + return Collections.emptyList(); + } +} diff --git a/org.simantics.district.imports.ui/src/org/simantics/district/imports/ui/CSVImportWizard.java b/org.simantics.district.imports.ui/src/org/simantics/district/imports/ui/CSVImportWizard.java new file mode 100644 index 00000000..97fa990a --- /dev/null +++ b/org.simantics.district.imports.ui/src/org/simantics/district/imports/ui/CSVImportWizard.java @@ -0,0 +1,38 @@ +package org.simantics.district.imports.ui; + +import java.io.IOException; +import java.nio.file.Path; + +import org.eclipse.jface.viewers.IStructuredSelection; +import org.eclipse.jface.wizard.Wizard; +import org.eclipse.ui.IImportWizard; +import org.eclipse.ui.IWorkbench; +import org.simantics.district.imports.DistrictImportUtils; + +public class CSVImportWizard extends Wizard implements IImportWizard { + + private CSVImportModel model; + + public CSVImportWizard() { + setWindowTitle("Import CSV data"); + } + + + @Override + public void init(IWorkbench workbench, IStructuredSelection selection) { + model = new CSVImportModel(); + addPage(new CSVImportWizardPage(model)); + } + + @Override + public boolean performFinish() { + Path csvFile = model.getSource(); + try { + DistrictImportUtils.importCSVAsLayer(csvFile); + } catch (IOException e) { + e.printStackTrace(); + } + return false; + } + +} diff --git a/org.simantics.district.imports.ui/src/org/simantics/district/imports/ui/CSVImportWizardPage.java b/org.simantics.district.imports.ui/src/org/simantics/district/imports/ui/CSVImportWizardPage.java new file mode 100644 index 00000000..85eeb942 --- /dev/null +++ b/org.simantics.district.imports.ui/src/org/simantics/district/imports/ui/CSVImportWizardPage.java @@ -0,0 +1,185 @@ +package org.simantics.district.imports.ui; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.eclipse.jface.layout.GridDataFactory; +import org.eclipse.jface.layout.TableColumnLayout; +import org.eclipse.jface.layout.TreeColumnLayout; +import org.eclipse.jface.viewers.ColumnViewerToolTipSupport; +import org.eclipse.jface.viewers.ColumnWeightData; +import org.eclipse.jface.viewers.IContentProvider; +import org.eclipse.jface.viewers.TableViewer; +import org.eclipse.jface.viewers.TreeViewer; +import org.eclipse.jface.viewers.Viewer; +import org.eclipse.jface.wizard.WizardPage; +import org.eclipse.swt.SWT; +import org.eclipse.swt.events.SelectionEvent; +import org.eclipse.swt.events.SelectionListener; +import org.eclipse.swt.layout.GridData; +import org.eclipse.swt.layout.GridLayout; +import org.eclipse.swt.widgets.Button; +import org.eclipse.swt.widgets.Combo; +import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Layout; +import org.eclipse.swt.widgets.Table; +import org.eclipse.swt.widgets.TableColumn; +import org.eclipse.swt.widgets.TableItem; +import org.simantics.district.imports.DistrictImportUtils.CSVHeader; +import org.simantics.utils.ui.widgets.FileOrDirectorySelectionWidget; +import org.simantics.utils.ui.widgets.FileSelectionListener; +import org.simantics.utils.ui.widgets.FileSelectionWidget; + +public class CSVImportWizardPage extends WizardPage implements FileSelectionListener { + + private CSVImportModel model; + private FileSelectionWidget csvSelection; + private Table headerTable; + private Button firstAsHeader; + private Combo delimiterCombo; + private TableColumnLayout tableColumnLayout; + private TableViewer tableViewer; + + protected CSVImportWizardPage(CSVImportModel model) { + super("Import CSV Data"); + this.model = model; + setMessage("Select CSV file to import"); + } + + @Override + public void createControl(Composite parent) { + Composite composite = new Composite(parent, SWT.NONE); + composite.setLayout(new GridLayout(1,false)); + GridDataFactory.fillDefaults().grab(true, true).align(SWT.FILL, SWT.FILL).applyTo(composite); + + csvSelection = new FileSelectionWidget(composite, "CSV file", SWT.OPEN); + csvSelection.addListener(this); + + delimiterCombo = new Combo(composite, SWT.DROP_DOWN | SWT.READ_ONLY); + delimiterCombo.setItems(model.getDelimiterFormats()); + delimiterCombo.addSelectionListener(new SelectionListener() { + + @Override + public void widgetSelected(SelectionEvent e) { + model.setDelimiterByLabel(delimiterCombo.getItem(delimiterCombo.getSelectionIndex())); + updateHeaders(); + } + + @Override + public void widgetDefaultSelected(SelectionEvent e) { + + } + }); + + firstAsHeader = new Button(composite, SWT.CHECK); + firstAsHeader.setText("Read first row as header"); + firstAsHeader.setSelection(model.getReadFirstAsHeader()); + firstAsHeader.addSelectionListener(new SelectionListener() { + + @Override + public void widgetSelected(SelectionEvent e) { + model.setReadFirstAsHeader(firstAsHeader.getSelection()); + updateHeaders(); + } + + @Override + public void widgetDefaultSelected(SelectionEvent e) { + + } + }); + + + Composite tableComposite = new Composite(composite, SWT.BORDER); + TreeColumnLayout treeColumnLayout = new TreeColumnLayout(); + tableComposite.setLayout(treeColumnLayout); + + tableViewer = new TableViewer(tableComposite, SWT.V_SCROLL | SWT.H_SCROLL); + ColumnViewerToolTipSupport.enableFor(tableViewer); + tableViewer.setContentProvider(new IContentProvider() { + + @Override + public void inputChanged(Viewer viewer, Object oldInput, Object newInput) { + + } + + @Override + public void dispose() { + + } + }); + + Table table = tableViewer.getTable(); + table.setHeaderVisible(true); + table.setLinesVisible(true); + + headerTable = new Table(tableComposite, SWT.NONE); + headerTable.setHeaderVisible(true); + headerTable.setLinesVisible(true); + GridDataFactory.fillDefaults().grab(true, true).align(SWT.FILL, SWT.FILL).applyTo(tableComposite); + + setControl(composite); + + updateHeaders(); + + setPageComplete(false); + } + + @Override + public void fileSelected(FileOrDirectorySelectionWidget source, String[] filename) { + String[] selection = csvSelection.getFilename(); + if (selection != null && selection.length > 0) { + Path csvFile = Paths.get(selection[0]); + if (!Files.exists(csvFile)) { + setErrorMessage("File " + csvFile.toAbsolutePath() + " does not exist"); + } else { + model.setSource(csvFile); + updateHeaders(); + setPageComplete(true); + } + } else { + setPageComplete(false); + } + } + + private void updateHeaders() { + headerTable.setRedraw(false); + for (int i = 0; i < headerTable.getColumns().length; i++) { + TableColumn column = headerTable.getColumns()[i]; + column.dispose(); + } + for (int i = 0; i < headerTable.getItemCount(); i++) { + TableItem item = headerTable.getItem(i); + item.dispose(); + } + try { + Map> headerAndValues = model.getHeaderAndRows(5); + + int coo = headerTable.getColumns().length; + int count = headerTable.getColumnCount(); + + for (Map.Entry> entr : headerAndValues.entrySet()) { + CSVHeader he = entr.getKey(); + int index = he.getIndex(); + TableColumn headerCol = new TableColumn(headerTable, SWT.NONE); + headerCol.setText(he.getHeader()); + + tableColumnLayout.setColumnData(headerCol, new ColumnWeightData(10)); + headerCol.pack(); + + for (String val : entr.getValue()) { + TableItem item = new TableItem(headerTable, SWT.NONE); + item.setText(val); + } + } + } catch (IOException e) { + setErrorMessage(e.getMessage()); + } + headerTable.setRedraw(true); + } + +} diff --git a/org.simantics.district.imports/.classpath b/org.simantics.district.imports/.classpath new file mode 100644 index 00000000..ac0cdbaf --- /dev/null +++ b/org.simantics.district.imports/.classpath @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/org.simantics.district.imports/.project b/org.simantics.district.imports/.project new file mode 100644 index 00000000..aebad524 --- /dev/null +++ b/org.simantics.district.imports/.project @@ -0,0 +1,34 @@ + + + org.simantics.district.imports + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.pde.ManifestBuilder + + + + + org.eclipse.pde.SchemaBuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.m2e.core.maven2Nature + org.eclipse.pde.PluginNature + org.eclipse.jdt.core.javanature + + diff --git a/org.simantics.district.imports/META-INF/MANIFEST.MF b/org.simantics.district.imports/META-INF/MANIFEST.MF new file mode 100644 index 00000000..50cc5d26 --- /dev/null +++ b/org.simantics.district.imports/META-INF/MANIFEST.MF @@ -0,0 +1,10 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Simantics District Imports +Bundle-SymbolicName: org.simantics.district.imports +Bundle-Version: 1.0.0.qualifier +Bundle-RequiredExecutionEnvironment: JavaSE-1.8 +Require-Bundle: org.simantics.db +Export-Package: org.simantics.district.imports +Bundle-ClassPath: lib/commons-csv-1.4.jar, + . diff --git a/org.simantics.district.imports/build.properties b/org.simantics.district.imports/build.properties new file mode 100644 index 00000000..78db4425 --- /dev/null +++ b/org.simantics.district.imports/build.properties @@ -0,0 +1,5 @@ +source.. = src/ +output.. = bin/ +bin.includes = META-INF/,\ + .,\ + lib/commons-csv-1.4.jar diff --git a/org.simantics.district.imports/lib/commons-csv-1.4-sources.jar b/org.simantics.district.imports/lib/commons-csv-1.4-sources.jar new file mode 100644 index 00000000..baf9b88f Binary files /dev/null and b/org.simantics.district.imports/lib/commons-csv-1.4-sources.jar differ diff --git a/org.simantics.district.imports/lib/commons-csv-1.4.jar b/org.simantics.district.imports/lib/commons-csv-1.4.jar new file mode 100644 index 00000000..e9eb0bd3 Binary files /dev/null and b/org.simantics.district.imports/lib/commons-csv-1.4.jar differ diff --git a/org.simantics.district.imports/pom.xml b/org.simantics.district.imports/pom.xml new file mode 100644 index 00000000..ed7ddaa2 --- /dev/null +++ b/org.simantics.district.imports/pom.xml @@ -0,0 +1,42 @@ + + 4.0.0 + org.simantics.district.imports + org.simantics.district.imports + 0.0.1-SNAPSHOT + eclipse-plugin + + + + UTF-8 + 0.25.0 + + + + + org.apache.commons + commons-csv + 1.4 + + + + + src + + + org.eclipse.tycho + tycho-maven-plugin + ${tycho-version} + true + + + maven-compiler-plugin + 3.5.1 + + 1.8 + 1.8 + + + + + \ No newline at end of file diff --git a/org.simantics.district.imports/src/org/simantics/district/imports/DistrictImportUtils.java b/org.simantics.district.imports/src/org/simantics/district/imports/DistrictImportUtils.java new file mode 100644 index 00000000..7c0d998c --- /dev/null +++ b/org.simantics.district.imports/src/org/simantics/district/imports/DistrictImportUtils.java @@ -0,0 +1,156 @@ +package org.simantics.district.imports; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.simantics.db.Resource; + +public class DistrictImportUtils { + + private DistrictImportUtils() { } + + + public static Resource importCSVAsLayer(Path csvFile) throws IOException { + + + try (CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(Files.newBufferedReader(csvFile))) { + Map header = parser.getHeaderMap(); + System.out.println(header); + } + return null; + } + + public static Map readCSVHeader(Path source, char delimiter, boolean firstAsHeader) throws IOException { + return readCSVHeader(source, CSVFormat.newFormat(delimiter), firstAsHeader); + } + + public static Map readCSVHeader(Path source, CSVFormat format, boolean firstAsHeader) throws IOException { + if (firstAsHeader) + format = format.withFirstRecordAsHeader(); + try (CSVParser parser = format.parse(Files.newBufferedReader(source))) { + return parser.getHeaderMap(); + } + } + + + public static Map getSupportedCSVDelimiterFormats() { + Map delimiters = new HashMap<>(); + delimiters.put("COMMA", ','); + delimiters.put("SEMICOMMA", ';'); + return delimiters; + } + + public static List> readRows(Path source, char delimiter, boolean firstAsHeader, int amount) throws IOException { + return readRows(source, CSVFormat.newFormat(delimiter), firstAsHeader, amount); + } + + public static List> readRows(Path source, CSVFormat format, boolean firstAsHeader, int amount) throws IOException { + if (firstAsHeader) + format = format.withFirstRecordAsHeader(); + try (CSVParser parser = format.parse(Files.newBufferedReader(source))) { + int start = 0; + List> results = new ArrayList<>(amount); + Iterator iter = parser.iterator(); + while (start < amount && iter.hasNext()) { + CSVRecord record = iter.next(); + results.add(record.toMap()); + start++; + } + return results; + } + } + + + public static Map> readCSVHeaderAndRows(Path source, char delimiter, boolean firstAsHeader, int amount) throws IOException { + Map> results = new HashMap<>(); + CSVFormat format = CSVFormat.newFormat(delimiter); + if (firstAsHeader) + format = format.withFirstRecordAsHeader(); + try (CSVParser parser = format.parse(Files.newBufferedReader(source))) { + Map headers = parser.getHeaderMap(); + if (headers != null && !headers.isEmpty()) { + for (int index = 0; index < headers.size(); index++) { + for (String head : headers.keySet()) { + results.put(new CSVHeader(head, index), new ArrayList<>()); + } + } + } + + Iterator records = parser.iterator(); + int rows = 0; + while (rows < amount && records.hasNext()) { + CSVRecord record = records.next(); + for (int j = 0; j < record.size(); j++) { + String value = record.get(j); + String header = Integer.toString(j); + CSVHeader csvHeader = new CSVHeader(header, j); + List vals = results.get(csvHeader); + if (vals == null) { + vals = new ArrayList<>(); + results.put(csvHeader, vals); + } + vals.add(value); + } + rows++; + } + } + return results; + } + + public static class CSVHeader { + + private final String header; + private final int index; + + public CSVHeader(String header, int index) { + this.header = header; + this.index = index; + } + + public String getHeader() { + return header; + } + + public int getIndex() { + return index; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((header == null) ? 0 : header.hashCode()); + result = prime * result + index; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + CSVHeader other = (CSVHeader) obj; + if (header == null) { + if (other.header != null) + return false; + } else if (!header.equals(other.header)) + return false; + if (index != other.index) + return false; + return true; + } + } + +} diff --git a/org.simantics.district.ui.feature/.project b/org.simantics.district.ui.feature/.project new file mode 100644 index 00000000..e8ca237e --- /dev/null +++ b/org.simantics.district.ui.feature/.project @@ -0,0 +1,17 @@ + + + org.simantics.district.ui.feature + + + + + + org.eclipse.pde.FeatureBuilder + + + + + + org.eclipse.pde.FeatureNature + + diff --git a/org.simantics.district.ui.feature/build.properties b/org.simantics.district.ui.feature/build.properties new file mode 100644 index 00000000..82ab19c6 --- /dev/null +++ b/org.simantics.district.ui.feature/build.properties @@ -0,0 +1 @@ +bin.includes = feature.xml diff --git a/org.simantics.district.ui.feature/feature.xml b/org.simantics.district.ui.feature/feature.xml new file mode 100644 index 00000000..e43314c8 --- /dev/null +++ b/org.simantics.district.ui.feature/feature.xml @@ -0,0 +1,37 @@ + + + + + [Enter Feature Description here.] + + + + [Enter Copyright Description here.] + + + + [Enter License Description here.] + + + + + + + + +