/*******************************************************************************
 * Copyright (c) 2007, 2010 Association for Decentralized Information Management
 * in Industry THTH ry.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *     VTT Technical Research Centre of Finland - initial API and implementation
 *******************************************************************************/
package org.simantics.db.layer0.util;

import gnu.trove.list.array.TIntArrayList;

import java.io.ByteArrayInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentSkipListSet;
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.commons.io.output.DeferredFileOutputStream;
import org.simantics.databoard.Bindings;
import org.simantics.databoard.Databoard;
import org.simantics.databoard.binding.mutable.Variant;
import org.simantics.databoard.serialization.Serializer;
import org.simantics.db.AsyncReadGraph;
import org.simantics.db.ReadGraph;
import org.simantics.db.Resource;
import org.simantics.db.Statement;
import org.simantics.db.common.StandardStatement;
import org.simantics.db.common.utils.NameUtils;
import org.simantics.db.exception.DatabaseException;
import org.simantics.db.exception.ValidationException;
import org.simantics.db.layer0.adapter.Instances;
import org.simantics.db.layer0.adapter.SubgraphAdvisor;
import org.simantics.db.layer0.adapter.SubgraphExtent;
import org.simantics.db.layer0.adapter.SubgraphExtent.Classifier;
import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
import org.simantics.db.procedure.AsyncProcedure;
import org.simantics.db.request.AsyncRead;
import org.simantics.db.request.Read;
import org.simantics.db.service.CollectionSupport;
import org.simantics.db.service.QueryControl;
import org.simantics.db.service.QueryControl.ControlProcedure;
import org.simantics.db.service.SerialisationSupport;
import org.simantics.graph.representation.External;
import org.simantics.graph.representation.Identity;
import org.simantics.graph.representation.Root;
import org.simantics.graph.representation.TransferableGraph1;
import org.simantics.graph.representation.Value;
import org.simantics.layer0.Layer0;
import org.simantics.operation.Layer0X;
import org.simantics.utils.datastructures.Pair;
import org.simantics.utils.datastructures.Triple;

/**
 * IsComposedOf objects are always in domain
 * Ordered set elements are always in domain
 * r is in domain if for all statements (s, IsRelatedTo, r) s is in domain
 *
 * IsWeaklyRelatedTo statements where subject is in domain are only accepted if object is in domain or has URI
 * 
 * @deprecated in favor of {@link TransferableGraphRequest2}
 */
public class TransferableGraphRequest implements Read<TransferableGraph1> {

	public static String LOG_FILE = "transferableGraph.log";
	final static private boolean LOG = false;
	final static private boolean DEBUG = false;
	final static private boolean PROFILE = false;
	
//	final private Collection<Pair<Resource, String>> roots;
//	private Collection<Resource> preExternals = Collections.emptyList();
//	private Resource model;
	
	private TransferableGraphConfiguration configuration;
	
	static DataOutput log;

	static {

		if (LOG) {
			try {
				FileOutputStream stream = new FileOutputStream(LOG_FILE);
				log = new DataOutputStream(stream);
			} catch (FileNotFoundException e) {
				e.printStackTrace();
			}
		}

	}
	
	private static void log(String line) {
		if (LOG) {
			try {
				log.writeUTF(line + "\n");
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
	}
	
	
	
	public TransferableGraphRequest(Collection<Pair<Resource, String>> roots, Resource model) {
		
		configuration = new TransferableGraphConfiguration();
		configuration.roots = roots;
		configuration.model = model;
		
//		this.roots = roots;
//		this.model = model;
		
	}

	
	public TransferableGraphRequest(Collection<Pair<Resource, String>> roots) {
		this(roots, null);
	}

	public TransferableGraphRequest(TransferableGraphConfiguration conf) {
		this.configuration = conf;
	}

	Layer0 l0;
	
	TIntArrayList inverses = new TIntArrayList();
	int statements[];
	int statementIndex = 0;
	Map<Resource, Integer> ids;
	Map<Resource, Variant> values;
	TIntArrayList externalParents = new TIntArrayList();
	ArrayList<String> externalNames = new ArrayList<String>(); 
//	ConcurrentLinkedQueue<Pair<Resource, byte[]>> values = new ConcurrentLinkedQueue<Pair<Resource, byte[]>>();
	int id = 0;
	int internalCount;
	
	int indent = 0;

	private Serializer variantSerializer;

	private boolean validateExternal(Resource r) {
		if(configuration.disallowedExternals != null) {
			System.err.println("validateExternal agains " + configuration.disallowedExternals);
			return !configuration.disallowedExternals.contains(r);
		}
		return true;
	}
	
	public int getId(ReadGraph graph, Resource r, Resource predicate) throws DatabaseException {
//	    for(int i=0;i<indent;++i)
//	        System.out.print("  ");
//	    System.out.println(NameUtils.getSafeName(graph, r));
		if(ids.containsKey(r)) {
		    int ret = ids.get(r);
		    if(ret == -1) {
		        for(int i=0;i<=indent;++i)
		            System.out.print("  ");
		        System.out.println("Cycle!!!"); // with " + GraphUtils.getReadableName(g, r));
		    }
			return ret;
		}
		else {
		    //ids.put(r, -1);
			Collection<Resource> parents = graph.getObjects(r, l0.PartOf);			
			if(parents.size() != 1) {
			    if(parents.size() == 0) {
			        Resource inv = graph.getPossibleObject(r, l0.InverseOf);
			        if(inv != null) {			
			            ++indent;
			            if(DEBUG) System.out.println("inverse " + NameUtils.getSafeName(graph, inv));
			            int invId = getId(graph, inv, null);
			            externalParents.add(invId);
			            --indent;
			            externalNames.add("@inverse");
			            ids.put(r, id);
			            return id++;
			        }
			    }
//			    for(Statement stat : graph.getStatements(r, b.IsWeaklyRelatedTo))
//			        System.out.println(NameUtils.getSafeName(graph, stat.getPredicate()) 
//			            + " -> " + NameUtils.getSafeName(graph, stat.getObject()));
//			    if(predicate != null) {
//			    	if(!graph.isSubrelationOf(predicate, b.IsRelatedTo)) {
//			    		return -2;
//			    	}
//			    }
				throw new ValidationException("Reference to external resource " 
						+ NameUtils.getSafeName(graph, r, true) + " without unique uri (" + parents.size() + " parents).");
			}
			for(Resource p : parents) {
//	            System.out.println("Parent " + NameUtils.getSafeName(graph, p)); 
			    ++indent;
	            if(!validateExternal(p)) throw new ValidationException("References to '" + graph.getURI(p) + "' are not allowed.");
				externalParents.add(getId(graph, p, null));
				--indent;
			}
//          System.out.println("Request named for " + GraphUtils.getReadableName(g, r)); 
//			String name = graph.getPossibleRelatedValue(r, b.HasName);
            externalNames.add((String)graph.getRelatedValue(r, l0.HasName));
//			if(name != null) externalNames.add(name);
//			else externalNames.add("@resource");
			ids.put(r, id);
			return id++;
		}
	}
	
	public void addId(ReadGraph graph, Resource r, Resource predicate) throws DatabaseException {
		statements[statementIndex++] = getId(graph, r, predicate); 
	}
	
	private void searchStatementPart(ReadGraph graph, final ConcurrentSkipListSet<Statement> statementSet, final ArrayList<Pair<Resource, Triple[]>> part, final Set<SubgraphExtent> extents) throws DatabaseException {

		final SubgraphExtent.Callback callback = new SubgraphExtent.Callback() {

			@Override
			public void statement(Statement statement, boolean accept) {
				if(accept) {
					statementSet.add(statement);
				}
			}
			
		};

		graph.syncRequest(new AsyncRead<Boolean>() {

		    @Override
		    public int threadHash() {
		    	return hashCode();
		    }
			
			@Override
			public void perform(AsyncReadGraph graph, AsyncProcedure<Boolean> procedure) {

				QueryControl control = graph.getService(QueryControl.class);

				int slice = (int)(part.size() / control.getAmountOfQueryThreads()) + 1;

				final Pair[] rootArray = (Pair[])part.toArray(new Pair[part.size()]);
				for(int i=0;i<control.getAmountOfQueryThreads();i++) {

					final int start = i * slice;
					final int end = Math.min(start + slice, rootArray.length);

					control.schedule(graph, i, new ControlProcedure() {

						@Override
						public void execute(AsyncReadGraph graph) {

							for(int index = start;index < end;index++) {

								final Pair<Resource, Triple[]> r = (Pair<Resource, Triple[]>)rootArray[index];

								final AtomicInteger position = new AtomicInteger(0);
								final SubgraphExtent.Classifier[] classifiers = new SubgraphExtent.Classifier[extents.size()];

								for(SubgraphExtent extent : extents) {

									extent.accept(graph, r.first, new AsyncProcedure<Classifier>() {

										public void execute(AsyncReadGraph graph, Classifier c) {

											int current = position.incrementAndGet();
											classifiers[current-1] = c;

											if(current == extents.size()) {

												for(Triple statement : r.second) {
													if(!r.first.isPersistent()) continue;
													for(Classifier classifier : classifiers) {
														classifier.classify(graph, new StandardStatement(r.first, (Resource)statement.first, (Resource)statement.second), (ExtentStatus)statement.third, callback);
													}
												}
												
//												graph.forEachDirectStatement(r, new AsyncMultiProcedure<Statement>() {
//
//													@Override
//													public void exception(AsyncReadGraph graph, Throwable throwable) {
//														throwable.printStackTrace();
//													}
//
//													@Override
//													public void execute(AsyncReadGraph graph, Statement statement) {
//													}
//
//													@Override
//													public void finished(AsyncReadGraph graph) {
//													}
//
//												});

											}

										}

										public void exception(AsyncReadGraph graph, Throwable throwable) {
											throwable.printStackTrace();
										}

									}, callback);

								}

							}

						}

					});

				}

				procedure.execute(graph, false);

			}

			@Override
			public int getFlags() {
				return 0;
			}

		});

	}

	class StatementSetBuilder {

		private int SLICE = 100000;
		
		final private CollectionSupport cs;
		final private LinkedList<Collection<Statement>> sets = new LinkedList<Collection<Statement>>();

		private Collection<Statement> current;
		
		StatementSetBuilder(CollectionSupport cs) {
			this.cs = cs;
			current = cs.createStatementList();
		}
		
		LinkedList<Collection<Statement>> get() {
			sets.add(current);
			return sets;
		}
		
		void add(Statement stm) {
			current.add(stm);
			if(current.size() == SLICE) {
				sets.add(current);
				current = cs.createStatementList();
			}
		}

		int size() {
			int result = 0;
			for(Collection<Statement> c : sets) result += c.size();
			return result;
		}
		
		
		void addAll(Collection<Statement> ss) {
			for(Statement s : ss) add(s);
		}
		
	}
	
	private void searchStatements(ReadGraph graph, ObjectInputStream composedInput, ObjectInputStream statementInput, ObjectInputStream valueInput, final Set<SubgraphExtent> extents) throws DatabaseException, IOException {

		CollectionSupport cs = graph.getService(CollectionSupport.class);
		
		ConcurrentSkipListSet<Statement> statementSet = new ConcurrentSkipListSet<Statement>();

		StatementSetBuilder builder = new StatementSetBuilder(cs);

		SerialisationSupport support = graph.getService(SerialisationSupport.class);

		while(composedInput.available() > 0) {

			int s = composedInput.readInt();
			Resource subject = support.getResource(s);

			int size = composedInput.readInt();
			for(int i=0;i<size;i++) {
				int p = composedInput.readInt();
				int o = composedInput.readInt();
				StandardStatement stm = new StandardStatement(subject, support.getResource(p), support.getResource(o));
//				if(DEBUG) System.out.println("composed " + NameUtils.toString(graph, stm));
				builder.add(stm);
			}
			
		}

		while(statementInput.available() > 0) {

			ArrayList<Pair<Resource, Triple[]>> stms = new ArrayList<Pair<Resource, Triple[]>>(); 

			int counter = 0;
			
			while(statementInput.available() > 0 && counter++ < 1000) {
			
				int s = statementInput.readInt();
				Resource subject = support.getResource(s);
				int size = statementInput.readInt();
				Triple[] list = new Triple[size];
				stms.add(Pair.make(subject, list));
				for(int i=0;i<size;i++) {
					int p = statementInput.readInt();
					int o = statementInput.readInt();
					if(DEBUG) {
						System.out.println("otherInput " + NameUtils.toString(graph, new StandardStatement(support.getResource(s), support.getResource(p), support.getResource(o))));
					}
					list[i] = new Triple<Resource, Resource, ExtentStatus>(support.getResource(p), support.getResource(o), ExtentStatus.EXTERNAL);
				}
			}
			
			searchStatementPart(graph, statementSet, stms, extents);
			
		}

		if(DEBUG) {
			for(Statement stm : statementSet)
				System.out.println("other " + NameUtils.toString(graph, stm));
		}
		
		builder.addAll(statementSet);
		
		if(DEBUG) System.out.println("total " + statementSet.size() + " statements found.");

		statementSet = null;
		
		LinkedList<Collection<Statement>> statementSets = builder.get();
		
		Map<Resource, Resource> inverses = cs.createMap(Resource.class);
		Set<Resource> predicateSet = cs.createSet();
		for(Collection<Statement> set : statementSets)
			for(Statement s : set) {
				Resource predicate = s.getPredicate();
				if(predicateSet.add(predicate)) {
					Resource inverse = graph.getPossibleInverse(predicate);
					inverses.put(predicate, inverse);
				}
			}

		predicateSet = null;
		
		statements = new int[4 * builder.size()];
		
        while(!statementSets.isEmpty()) {
        	Collection<Statement> set = statementSets.pop();
			for(Statement s : set) {
				Resource subject = s.getSubject();
				Resource predicate = s.getPredicate();
				Resource object = s.getObject();
				int subjectId = ids.get(subject);
				int objectId = getId(graph, object, predicate);
				// The statement can be denied still
				if(objectId != -2) {
					statements[statementIndex++] = subjectId;
					addId(graph, predicate, null);
					Resource inverse = inverses.get(predicate);
					if(inverse != null) {
						if(LOG) log("[STM] " + subject.getResourceId() + ", " + predicate.getResourceId() + ", " + inverse.getResourceId() + ", " + object.getResourceId());
						addId(graph, inverse, null);
					} else {
						if(LOG) log("[STM] " + subject.getResourceId() + ", " + predicate.getResourceId() + ", -1, " + object.getResourceId());
						statements[statementIndex++] = -1;
					}
					statements[statementIndex++] = objectId;
					
					
					
				} else {
					System.out.println("denied");
				}
			}
		}
		
		inverses = null;
		
		while(valueInput.available() > 0) {

			int s = valueInput.readInt();
			Resource subject = support.getResource(s);
			int valueSize = valueInput.readInt();
			byte[] value = new byte[valueSize];
			valueInput.readFully(value);
			Variant variant = (Variant) variantSerializer.deserialize(value);
			values.put(subject, variant);
			
		}
		
	}
	
	public void setExternals(Collection<Resource> rs) {
		configuration.externals = rs;
	}
	
	@Override
	public TransferableGraph1 perform(ReadGraph graph) throws DatabaseException {

		this.variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);

		this.l0 = Layer0.getInstance(graph);
		Layer0X L0X = Layer0X.getInstance(graph);
		
		long total = System.nanoTime();

    	long startupTime = System.nanoTime();

    	CollectionSupport cs = graph.getService(CollectionSupport.class);
    	
    	ids = cs.createMap(Integer.class);
    	values = cs.createMap(byte[].class);
    	
		ArrayList<Resource> rootResources = new ArrayList<Resource>();
		for(Pair<Resource, String> p : configuration.roots) rootResources.add(p.first);

//		if(model == null) {
//			SimulationResource SIMU = SimulationResource.getInstance(graph);
//			for(Resource root : rootResources) {
//				if(graph.isInstanceOf(root, SIMU.Model)) {
//					model = root;
//					break;
//				}
//				model = graph.syncRequest(new TypedParent(root, SIMU.Model));
//				if(model != null) break;
//			}
//		}

		Map<Resource, ExtentStatus> preStatus = new HashMap<Resource, ExtentStatus>();
		
		for(Resource root : rootResources) {
			Resource name = graph.getPossibleObject(root, l0.HasName);
			if(name != null) {
				preStatus.put(name, ExtentStatus.EXCLUDED);
			}
		}
		
		for(Resource r : configuration.externals) preStatus.put(r, ExtentStatus.EXTERNAL); 

		Set<SubgraphExtent> extents = configuration.extents;
		if(extents == null) {
			
			Instances extentInstances = graph.adapt(L0X.SubgraphExtent, Instances.class);
			Collection<Resource> extentResources = extentInstances.find(graph, configuration.model);
			extents = new HashSet<SubgraphExtent>();
			for(Resource r : extentResources) {
				extents.add(graph.getPossibleAdapter(r, SubgraphExtent.class));
			}

			if(DEBUG) {
				System.out.println("Extents " + extents.size());
				for(Resource extent : extentResources) System.out.println("-" + NameUtils.getSafeName(graph, extent));
			}
			
		}
		
		Set<SubgraphAdvisor> advisors = configuration.advisors;
		if(advisors == null) {

			Instances advisorInstances = graph.adapt(L0X.SubgraphAdvisor, Instances.class);
			Collection<Resource> advisorResources = advisorInstances.find(graph, configuration.model);
			advisors = new HashSet<SubgraphAdvisor>();
			for(Resource r : advisorResources) {
				advisors.add(graph.getPossibleAdapter(r, SubgraphAdvisor.class));
			}

			if(DEBUG) {
				System.out.println("Advisors " + advisors.size());
				for(Resource advisor : advisorResources) System.out.println("-" + NameUtils.getSafeName(graph, advisor));
			}
			
		}

    	long startupTimeEnd = System.nanoTime();

		long domainTime = System.nanoTime();
		
		String composedStatements = "composed" + UUID.randomUUID().toString();
		String otherStatements = "other" + UUID.randomUUID().toString();
		String valueFileName = "value" + UUID.randomUUID().toString();
		
        File composedStatementsFile = new File(composedStatements);
        File otherStatementsFile = new File(otherStatements);
        File valueFile = new File(valueFileName);
        
        if(DEBUG)
        	System.out.println("getDomain writes " + composedStatementsFile.getAbsolutePath());

        try {
        
        	DeferredFileOutputStream composedStatementsStream = new DeferredFileOutputStream(5 * 1024*1024, composedStatementsFile);
        	DeferredFileOutputStream otherStatementsStream = new DeferredFileOutputStream(1024*1024, otherStatementsFile);
        	DeferredFileOutputStream valueStream = new DeferredFileOutputStream(1024*1024, valueFile);
        	
        	ObjectOutputStream composedStatementsOutput = new ObjectOutputStream(composedStatementsStream);
        	ObjectOutputStream otherStatementsOutput = new ObjectOutputStream(otherStatementsStream);
        	ObjectOutputStream valueOutput = new ObjectOutputStream(valueStream);

        	Subgraphs.getDomain(graph, ids, rootResources, preStatus, advisors, composedStatementsOutput, otherStatementsOutput, valueOutput);
        	id = ids.size();

        	composedStatementsOutput.flush();
        	otherStatementsOutput.flush();
        	valueOutput.flush();
        	composedStatementsStream.close();
        	otherStatementsStream.close();
        	valueStream.close();
		
        	long domainTimeEnd = System.nanoTime();

        	long fileTime = System.nanoTime();

        	internalCount = id;

        	ids.put(graph.getRootLibrary(), id++);
        	externalNames.add("http:/");
        	externalParents.add(-1);

//        	if(model == null) return null;

        	InputStream composedStatementsInputStream = null;
        	InputStream otherStatementsInputStream = null;
        	InputStream valueInputStream = null;
        	
        	if(composedStatementsStream.isInMemory()) {
        		composedStatementsInputStream = new ByteArrayInputStream(composedStatementsStream.getData());
        	} else {
        		composedStatementsInputStream = new FileInputStream(composedStatementsFile);
        	}
        	
        	if(otherStatementsStream.isInMemory()) {
        		otherStatementsInputStream = new ByteArrayInputStream(otherStatementsStream.getData());
        	} else {
        		otherStatementsInputStream = new FileInputStream(otherStatementsFile);
        	}

        	if(valueStream.isInMemory()) {
        		valueInputStream = new ByteArrayInputStream(valueStream.getData());
        	} else {
        		valueInputStream = new FileInputStream(valueFile);
        	}

        	composedStatementsStream = null;
        	otherStatementsStream = null;
        	valueStream = null;
        	
        	ObjectInputStream composedStatementsInput = new ObjectInputStream(composedStatementsInputStream);
        	ObjectInputStream otherStatementsInput = new ObjectInputStream(otherStatementsInputStream);
        	ObjectInputStream valueInput = new ObjectInputStream(valueInputStream);

        	long fileTimeEnd = System.nanoTime();

        	long statementTime = System.nanoTime();
        	searchStatements(graph, composedStatementsInput, otherStatementsInput, valueInput, extents);
        	long statementTimeEnd = System.nanoTime();

        	//		domainParts = null;

        	long buildTime = System.nanoTime();

        	int resourceCount = ids.size();

        	Identity[] identityArray;
        	{ // Identities
        		ArrayList<Identity> identities = new ArrayList<Identity>();

        		for(Pair<Resource, String> r : configuration.roots) {
        			Resource type = graph.getPossibleType(r.first, l0.Entity);
        			if(type == null) type = l0.Entity;
        			identities.add(new Identity(
        					ids.get(r.first),
        					new Root(r.second, graph.getURI(type))
        			));
        		}

        		int internalsPlusExternals = ids.size();		
        		for(int i = internalCount; i < internalsPlusExternals ; i++) {
        			int parent = externalParents.get(i - internalCount);
        			String name = externalNames.get(i - internalCount);
        			identities.add(new Identity(
        					i, 
        					new External(parent, name)
        			));
        		}
        		identityArray = identities.toArray(new Identity[identities.size()]);
        	}            

        	Value[] valueArray = new Value[values.size()];
        	{ // Values
        		int index = 0;
        		for(Map.Entry<Resource, Variant> entry : values.entrySet()) {
        			if(LOG) log("[VALUE] " + entry.getKey().getResourceId());
        			int r = getId(graph, entry.getKey(), null);
        			valueArray[index++] = new Value(r, entry.getValue());
        		}
        	}
        	ids = null;
        	values = null;

        	long buildTimeEnd = System.nanoTime();

        	TransferableGraph1 result = 
        	    new TransferableGraph1(resourceCount, 
        	            identityArray, 
        	            statements, 
        	            valueArray);

        	if(DEBUG) {
        		System.out.println("transferable graph content: " + result);
        	}

        	long totalEnd = System.nanoTime();

        	if(PROFILE) {	
        		System.out.println("startup in " + 1e-9*(startupTimeEnd - startupTime) + "s.");
        		System.out.println("domain was found in " + 1e-9*(domainTimeEnd - domainTime) + "s.");
        		System.out.println("files were written in " + 1e-9*(fileTimeEnd - fileTime) + "s.");
        		System.out.println("statements were found in " + 1e-9*(statementTimeEnd - statementTime) + "s.");
        		System.out.println("tg was built in " + 1e-9*(buildTimeEnd - buildTime) + "s.");
        		System.out.println("total time for building subgraph was " + 1e-9*(totalEnd-total) + "s.");
        	}        	

        	return result;
        } catch (IOException e) {
        	e.printStackTrace();
        }
		
        return null;
		
	}
	
}