tests/org.simantics.db.tests/src/org/simantics/db/tests/api/story/misc/TextIndexingTest.java

   1 /*******************************************************************************
   2  * Copyright (c) 2007, 2010 Association for Decentralized Information Management
   3  * in Industry THTH ry.
   4  * All rights reserved. This program and the accompanying materials
   5  * are made available under the terms of the Eclipse Public License v1.0
   6  * which accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *     VTT Technical Research Centre of Finland - initial API and implementation
  11  *******************************************************************************/
  12 package org.simantics.db.tests.api.story.misc;
  13
  14 import java.io.BufferedInputStream;
  15 import java.io.FileInputStream;
  16 import java.io.FileNotFoundException;
  17 import java.io.IOException;
  18 import java.io.InputStream;
  19 import java.util.Arrays;
  20
  21 import org.junit.After;
  22 import org.junit.Before;
  23 import org.junit.Test;
  24 import org.simantics.db.Resource;
  25 import org.simantics.db.Session;
  26 import org.simantics.db.WriteGraph;
  27 import org.simantics.db.common.request.WriteRequest;
  28 import org.simantics.db.exception.DatabaseException;
  29 import org.simantics.db.service.LifecycleSupport;
  30 import org.simantics.db.testing.annotation.Fails;
  31 import org.simantics.db.testing.common.Tests;
  32 import org.simantics.db.tests.common.Configuration;
  33 import org.simantics.layer0.Layer0;
  34
  35 public class TextIndexingTest {
  36
  37         static String host = Configuration.get().host;
  38         static int port = Configuration.get().port;
  39         static String dataFileName = Configuration.get().textIndexingFile;
  40         Session session;
  41         Text text;
  42         Resource root;
  43         Resource[] relations = new Resource[128];
  44
  45         @Before
  46         public void setUp() throws Exception {
  47 //              SessionFactory factory = new SessionFactory(host, port);
  48 //              session = factory.create();
  49         session = Tests.getTestHandler().getSession();
  50         }
  51
  52         @After
  53         public void tearDown() throws Exception {
  54         LifecycleSupport support = session.getService(LifecycleSupport.class);
  55         support.close();
  56         }
  57
  58         class SetUp extends WriteRequest {
  59
  60                 @Override
  61                 public void perform(WriteGraph g) throws DatabaseException {
  62                         root = g.newResource();
  63                         Layer0 l0 = Layer0.getInstance(g);
  64                         for(int i=0;i<128;++i) {
  65                                 relations[i] = g.newResource();
  66                                 Resource inv = g.newResource();
  67                                 g.claim(relations[i], l0.InverseOf, inv);
  68                                 g.claim(relations[i], l0.SubrelationOf, l0.IsRelatedTo);
  69                                 g.claim(inv, l0.SubrelationOf, l0.IsWeaklyRelatedTo);
  70                         }
  71                 }
  72
  73         }
  74
  75         boolean finished = false;
  76
  77         class WriteTrie extends WriteRequest {
  78
  79                 int count = 0;
  80                 int newWords = 0;
  81
  82                 @Override
  83                 public void perform(WriteGraph g) throws DatabaseException {
  84
  85                         try {
  86
  87                                 //System.out.println("T");
  88                                 byte[] word;
  89                                 Layer0 b = Layer0.getInstance(g);
  90                                 long startTime = System.nanoTime();
  91                                 while( (word = text.getWord()) != null ) {
  92
  93                                         if(word == PEND)
  94                                                 continue;
  95
  96                                         Resource cur = root;
  97                                         for(int i=0;i<word.length;++i) {
  98                                                 Resource relation = relations[word[i]];
  99                                                 Resource temp = g.getPossibleObject(cur, relation);
 100                                                 if(temp == null) {
 101                                                         temp = g.newResource();
 102                                                         g.claim(cur, relation, temp);
 103                                                 }
 104                                                 cur = temp;
 105                                         }
 106
 107                                         Resource name = g.getPossibleObject(cur, b.HasName);
 108                                         if(name == null) {
 109                                                 name = g.newResource();
 110                                                 g.claimValue(name, new String(word));
 111                                                 g.claim(cur, b.HasName, name);
 112                                                 ++newWords;
 113                                         }
 114
 115                                         ++count;
 116                                         if((count % 2000) == 0)
 117                                                 System.out.println(count + ", " + newWords);
 118                                 }
 119                                 long endTime = System.nanoTime();
 120                                 System.out.println("End");
 121                                 System.out.println(count + " words, " + newWords + " unique words");
 122                                 System.out.println((endTime-startTime)*1e-6/count + " ms / word");
 123
 124                                 finished = true;
 125
 126                         } catch (IOException e) {
 127                                 e.printStackTrace();
 128                         }
 129
 130                 }
 131
 132         }
 133
 134         @Fails
 135         @Test
 136         public void test() {
 137                 try {
 138                         text = new Text(new BufferedInputStream(new FileInputStream(dataFileName)));
 139                         session.syncRequest(new SetUp());
 140                         System.out.println("Start");
 141                         WriteTrie wt = new WriteTrie();
 142                         while(!finished)
 143                                 session.syncRequest(wt);
 144                         System.out.println("Ok");
 145                 } catch (FileNotFoundException e) {
 146                         // TODO Auto-generated catch block
 147                         e.printStackTrace();
 148                 } catch (DatabaseException e) {
 149                         e.printStackTrace();
 150                 }
 151         }
 152
 153         final static byte[] PEND = new byte[0];
 154
 155         static class Text {
 156                 InputStream s;
 157                 byte[] temp = new byte[1024];
 158
 159                 public Text(InputStream s) {
 160                         this.s = s;
 161                 }
 162
 163                 byte[] getWord() throws IOException {
 164                         int b;
 165                         while( !Character.isLetter(b = s.read()) && b != -1 && b != '\n');
 166                         if(b == -1)
 167                                 return null;
 168                         if(b == '\n')
 169                                 return PEND;
 170                         int i=0;
 171                         while( Character.isLetter(b) ) {
 172                                 temp[i] = (byte)Character.toLowerCase(b);
 173                                 ++i;
 174                                 b = s.read();
 175                         }
 176                         return Arrays.copyOf(temp, i);
 177                 }
 178         }
 179
 180 }