bundles/org.simantics.spreadsheet.common/src/org/simantics/spreadsheet/common/expression/lexer/Lexer.java

   1 /*******************************************************************************\r
   2  * Copyright (c) 2007, 2010 Association for Decentralized Information Management\r
   3  * in Industry THTH ry.\r
   4  * All rights reserved. This program and the accompanying materials\r
   5  * are made available under the terms of the Eclipse Public License v1.0\r
   6  * which accompanies this distribution, and is available at\r
   7  * http://www.eclipse.org/legal/epl-v10.html\r
   8  *\r
   9  * Contributors:\r
  10  *     VTT Technical Research Centre of Finland - initial API and implementation\r
  11  *******************************************************************************/\r
  12 /* This file was generated by SableCC (http://www.sablecc.org/). */\r
  13 \r
  14 package org.simantics.spreadsheet.common.expression.lexer;\r
  15 \r
  16 import java.io.*;\r
  17 import org.simantics.spreadsheet.common.expression.node.*;\r
  18 \r
  19 @SuppressWarnings("nls")\r
  20 public class Lexer\r
  21 {\r
  22     protected Token token;\r
  23     protected State state = State.INITIAL;\r
  24 \r
  25     private PushbackReader in;\r
  26     private int line;\r
  27     private int pos;\r
  28     private boolean cr;\r
  29     private boolean eof;\r
  30     private final StringBuffer text = new StringBuffer();\r
  31 \r
  32     @SuppressWarnings("unused")\r
  33     protected void filter() throws LexerException, IOException\r
  34     {\r
  35         // Do nothing\r
  36     }\r
  37 \r
  38     public Lexer(@SuppressWarnings("hiding") PushbackReader in)\r
  39     {\r
  40         this.in = in;\r
  41     }\r
  42  \r
  43     public Token peek() throws LexerException, IOException\r
  44     {\r
  45         while(this.token == null)\r
  46         {\r
  47             this.token = getToken();\r
  48             filter();\r
  49         }\r
  50 \r
  51         return this.token;\r
  52     }\r
  53 \r
  54     public Token next() throws LexerException, IOException\r
  55     {\r
  56         while(this.token == null)\r
  57         {\r
  58             this.token = getToken();\r
  59             filter();\r
  60         }\r
  61 \r
  62         Token result = this.token;\r
  63         this.token = null;\r
  64         return result;\r
  65     }\r
  66 \r
  67     protected Token getToken() throws IOException, LexerException\r
  68     {\r
  69         int dfa_state = 0;\r
  70 \r
  71         int start_pos = this.pos;\r
  72         int start_line = this.line;\r
  73 \r
  74         int accept_state = -1;\r
  75         int accept_token = -1;\r
  76         int accept_length = -1;\r
  77         int accept_pos = -1;\r
  78         int accept_line = -1;\r
  79 \r
  80         @SuppressWarnings("hiding") int[][][] gotoTable = Lexer.gotoTable[this.state.id()];\r
  81         @SuppressWarnings("hiding") int[] accept = Lexer.accept[this.state.id()];\r
  82         this.text.setLength(0);\r
  83 \r
  84         while(true)\r
  85         {\r
  86             int c = getChar();\r
  87 \r
  88             if(c != -1)\r
  89             {\r
  90                 switch(c)\r
  91                 {\r
  92                 case 10:\r
  93                     if(this.cr)\r
  94                     {\r
  95                         this.cr = false;\r
  96                     }\r
  97                     else\r
  98                     {\r
  99                         this.line++;\r
 100                         this.pos = 0;\r
 101                     }\r
 102                     break;\r
 103                 case 13:\r
 104                     this.line++;\r
 105                     this.pos = 0;\r
 106                     this.cr = true;\r
 107                     break;\r
 108                 default:\r
 109                     this.pos++;\r
 110                     this.cr = false;\r
 111                     break;\r
 112                 }\r
 113 \r
 114                 this.text.append((char) c);\r
 115 \r
 116                 do\r
 117                 {\r
 118                     int oldState = (dfa_state < -1) ? (-2 -dfa_state) : dfa_state;\r
 119 \r
 120                     dfa_state = -1;\r
 121 \r
 122                     int[][] tmp1 =  gotoTable[oldState];\r
 123                     int low = 0;\r
 124                     int high = tmp1.length - 1;\r
 125 \r
 126                     while(low <= high)\r
 127                     {\r
 128                         int middle = (low + high) / 2;\r
 129                         int[] tmp2 = tmp1[middle];\r
 130 \r
 131                         if(c < tmp2[0])\r
 132                         {\r
 133                             high = middle - 1;\r
 134                         }\r
 135                         else if(c > tmp2[1])\r
 136                         {\r
 137                             low = middle + 1;\r
 138                         }\r
 139                         else\r
 140                         {\r
 141                             dfa_state = tmp2[2];\r
 142                             break;\r
 143                         }\r
 144                     }\r
 145                 }while(dfa_state < -1);\r
 146             }\r
 147             else\r
 148             {\r
 149                 dfa_state = -1;\r
 150             }\r
 151 \r
 152             if(dfa_state >= 0)\r
 153             {\r
 154                 if(accept[dfa_state] != -1)\r
 155                 {\r
 156                     accept_state = dfa_state;\r
 157                     accept_token = accept[dfa_state];\r
 158                     accept_length = this.text.length();\r
 159                     accept_pos = this.pos;\r
 160                     accept_line = this.line;\r
 161                 }\r
 162             }\r
 163             else\r
 164             {\r
 165                 if(accept_state != -1)\r
 166                 {\r
 167                     switch(accept_token)\r
 168                     {\r
 169                     case 0:\r
 170                         {\r
 171                             @SuppressWarnings("hiding") Token token = new0(\r
 172                                 start_line + 1,\r
 173                                 start_pos + 1);\r
 174                             pushBack(accept_length);\r
 175                             this.pos = accept_pos;\r
 176                             this.line = accept_line;\r
 177                             return token;\r
 178                         }\r
 179                     case 1:\r
 180                         {\r
 181                             @SuppressWarnings("hiding") Token token = new1(\r
 182                                 start_line + 1,\r
 183                                 start_pos + 1);\r
 184                             pushBack(accept_length);\r
 185                             this.pos = accept_pos;\r
 186                             this.line = accept_line;\r
 187                             return token;\r
 188                         }\r
 189                     case 2:\r
 190                         {\r
 191                             @SuppressWarnings("hiding") Token token = new2(\r
 192                                 start_line + 1,\r
 193                                 start_pos + 1);\r
 194                             pushBack(accept_length);\r
 195                             this.pos = accept_pos;\r
 196                             this.line = accept_line;\r
 197                             return token;\r
 198                         }\r
 199                     case 3:\r
 200                         {\r
 201                             @SuppressWarnings("hiding") Token token = new3(\r
 202                                 start_line + 1,\r
 203                                 start_pos + 1);\r
 204                             pushBack(accept_length);\r
 205                             this.pos = accept_pos;\r
 206                             this.line = accept_line;\r
 207                             return token;\r
 208                         }\r
 209                     case 4:\r
 210                         {\r
 211                             @SuppressWarnings("hiding") Token token = new4(\r
 212                                 start_line + 1,\r
 213                                 start_pos + 1);\r
 214                             pushBack(accept_length);\r
 215                             this.pos = accept_pos;\r
 216                             this.line = accept_line;\r
 217                             return token;\r
 218                         }\r
 219                     case 5:\r
 220                         {\r
 221                             @SuppressWarnings("hiding") Token token = new5(\r
 222                                 start_line + 1,\r
 223                                 start_pos + 1);\r
 224                             pushBack(accept_length);\r
 225                             this.pos = accept_pos;\r
 226                             this.line = accept_line;\r
 227                             return token;\r
 228                         }\r
 229                     case 6:\r
 230                         {\r
 231                             @SuppressWarnings("hiding") Token token = new6(\r
 232                                 start_line + 1,\r
 233                                 start_pos + 1);\r
 234                             pushBack(accept_length);\r
 235                             this.pos = accept_pos;\r
 236                             this.line = accept_line;\r
 237                             return token;\r
 238                         }\r
 239                     case 7:\r
 240                         {\r
 241                             @SuppressWarnings("hiding") Token token = new7(\r
 242                                 start_line + 1,\r
 243                                 start_pos + 1);\r
 244                             pushBack(accept_length);\r
 245                             this.pos = accept_pos;\r
 246                             this.line = accept_line;\r
 247                             return token;\r
 248                         }\r
 249                     case 8:\r
 250                         {\r
 251                             @SuppressWarnings("hiding") Token token = new8(\r
 252                                 start_line + 1,\r
 253                                 start_pos + 1);\r
 254                             pushBack(accept_length);\r
 255                             this.pos = accept_pos;\r
 256                             this.line = accept_line;\r
 257                             return token;\r
 258                         }\r
 259                     case 9:\r
 260                         {\r
 261                             @SuppressWarnings("hiding") Token token = new9(\r
 262                                 start_line + 1,\r
 263                                 start_pos + 1);\r
 264                             pushBack(accept_length);\r
 265                             this.pos = accept_pos;\r
 266                             this.line = accept_line;\r
 267                             return token;\r
 268                         }\r
 269                     case 10:\r
 270                         {\r
 271                             @SuppressWarnings("hiding") Token token = new10(\r
 272                                 start_line + 1,\r
 273                                 start_pos + 1);\r
 274                             pushBack(accept_length);\r
 275                             this.pos = accept_pos;\r
 276                             this.line = accept_line;\r
 277                             return token;\r
 278                         }\r
 279                     case 11:\r
 280                         {\r
 281                             @SuppressWarnings("hiding") Token token = new11(\r
 282                                 start_line + 1,\r
 283                                 start_pos + 1);\r
 284                             pushBack(accept_length);\r
 285                             this.pos = accept_pos;\r
 286                             this.line = accept_line;\r
 287                             return token;\r
 288                         }\r
 289                     case 12:\r
 290                         {\r
 291                             @SuppressWarnings("hiding") Token token = new12(\r
 292                                 getText(accept_length),\r
 293                                 start_line + 1,\r
 294                                 start_pos + 1);\r
 295                             pushBack(accept_length);\r
 296                             this.pos = accept_pos;\r
 297                             this.line = accept_line;\r
 298                             return token;\r
 299                         }\r
 300                     case 13:\r
 301                         {\r
 302                             @SuppressWarnings("hiding") Token token = new13(\r
 303                                 getText(accept_length),\r
 304                                 start_line + 1,\r
 305                                 start_pos + 1);\r
 306                             pushBack(accept_length);\r
 307                             this.pos = accept_pos;\r
 308                             this.line = accept_line;\r
 309                             return token;\r
 310                         }\r
 311                     case 14:\r
 312                         {\r
 313                             @SuppressWarnings("hiding") Token token = new14(\r
 314                                 getText(accept_length),\r
 315                                 start_line + 1,\r
 316                                 start_pos + 1);\r
 317                             pushBack(accept_length);\r
 318                             this.pos = accept_pos;\r
 319                             this.line = accept_line;\r
 320                             return token;\r
 321                         }\r
 322                     case 15:\r
 323                         {\r
 324                             @SuppressWarnings("hiding") Token token = new15(\r
 325                                 getText(accept_length),\r
 326                                 start_line + 1,\r
 327                                 start_pos + 1);\r
 328                             pushBack(accept_length);\r
 329                             this.pos = accept_pos;\r
 330                             this.line = accept_line;\r
 331                             return token;\r
 332                         }\r
 333                     case 16:\r
 334                         {\r
 335                             @SuppressWarnings("hiding") Token token = new16(\r
 336                                 getText(accept_length),\r
 337                                 start_line + 1,\r
 338                                 start_pos + 1);\r
 339                             pushBack(accept_length);\r
 340                             this.pos = accept_pos;\r
 341                             this.line = accept_line;\r
 342                             return token;\r
 343                         }\r
 344                     case 17:\r
 345                         {\r
 346                             @SuppressWarnings("hiding") Token token = new17(\r
 347                                 getText(accept_length),\r
 348                                 start_line + 1,\r
 349                                 start_pos + 1);\r
 350                             pushBack(accept_length);\r
 351                             this.pos = accept_pos;\r
 352                             this.line = accept_line;\r
 353                             return token;\r
 354                         }\r
 355                     }\r
 356                 }\r
 357                 else\r
 358                 {\r
 359                     if(this.text.length() > 0)\r
 360                     {\r
 361                         throw new LexerException(\r
 362                             "[" + (start_line + 1) + "," + (start_pos + 1) + "]" +\r
 363                             " Unknown token: " + this.text);\r
 364                     }\r
 365 \r
 366                     @SuppressWarnings("hiding") EOF token = new EOF(\r
 367                         start_line + 1,\r
 368                         start_pos + 1);\r
 369                     return token;\r
 370                 }\r
 371             }\r
 372         }\r
 373     }\r
 374 \r
 375     Token new0(@SuppressWarnings("hiding") int line, @SuppressWarnings("hiding") int pos) { return new TLessThan(line, pos); }\r
 376     Token new1(@SuppressWarnings("hiding") int line, @SuppressWarnings("hiding") int pos) { return new TGreaterThan(line, pos); }\r
 377     Token new2(@SuppressWarnings("hiding") int line, @SuppressWarnings("hiding") int pos) { return new TEqual(line, pos); }\r
 378     Token new3(@SuppressWarnings("hiding") int line, @SuppressWarnings("hiding") int pos) { return new TPlus(line, pos); }\r
 379     Token new4(@SuppressWarnings("hiding") int line, @SuppressWarnings("hiding") int pos) { return new TMinus(line, pos); }\r
 380     Token new5(@SuppressWarnings("hiding") int line, @SuppressWarnings("hiding") int pos) { return new TMult(line, pos); }\r
 381     Token new6(@SuppressWarnings("hiding") int line, @SuppressWarnings("hiding") int pos) { return new TDiv(line, pos); }\r
 382     Token new7(@SuppressWarnings("hiding") int line, @SuppressWarnings("hiding") int pos) { return new TAmpersand(line, pos); }\r
 383     Token new8(@SuppressWarnings("hiding") int line, @SuppressWarnings("hiding") int pos) { return new TComma(line, pos); }\r
 384     Token new9(@SuppressWarnings("hiding") int line, @SuppressWarnings("hiding") int pos) { return new TColon(line, pos); }\r
 385     Token new10(@SuppressWarnings("hiding") int line, @SuppressWarnings("hiding") int pos) { return new TLPar(line, pos); }\r
 386     Token new11(@SuppressWarnings("hiding") int line, @SuppressWarnings("hiding") int pos) { return new TRPar(line, pos); }\r
 387     Token new12(@SuppressWarnings("hiding") String text, @SuppressWarnings("hiding") int line, @SuppressWarnings("hiding") int pos) { return new TFunc(text, line, pos); }\r
 388     Token new13(@SuppressWarnings("hiding") String text, @SuppressWarnings("hiding") int line, @SuppressWarnings("hiding") int pos) { return new TCell(text, line, pos); }\r
 389     Token new14(@SuppressWarnings("hiding") String text, @SuppressWarnings("hiding") int line, @SuppressWarnings("hiding") int pos) { return new TNumber(text, line, pos); }\r
 390     Token new15(@SuppressWarnings("hiding") String text, @SuppressWarnings("hiding") int line, @SuppressWarnings("hiding") int pos) { return new TString(text, line, pos); }\r
 391     Token new16(@SuppressWarnings("hiding") String text, @SuppressWarnings("hiding") int line, @SuppressWarnings("hiding") int pos) { return new TNewLine(text, line, pos); }\r
 392     Token new17(@SuppressWarnings("hiding") String text, @SuppressWarnings("hiding") int line, @SuppressWarnings("hiding") int pos) { return new TBlank(text, line, pos); }\r
 393 \r
 394     private int getChar() throws IOException\r
 395     {\r
 396         if(this.eof)\r
 397         {\r
 398             return -1;\r
 399         }\r
 400 \r
 401         int result = this.in.read();\r
 402 \r
 403         if(result == -1)\r
 404         {\r
 405             this.eof = true;\r
 406         }\r
 407 \r
 408         return result;\r
 409     }\r
 410 \r
 411     private void pushBack(int acceptLength) throws IOException\r
 412     {\r
 413         int length = this.text.length();\r
 414         for(int i = length - 1; i >= acceptLength; i--)\r
 415         {\r
 416             this.eof = false;\r
 417 \r
 418             this.in.unread(this.text.charAt(i));\r
 419         }\r
 420     }\r
 421 \r
 422     protected void unread(@SuppressWarnings("hiding") Token token) throws IOException\r
 423     {\r
 424         @SuppressWarnings("hiding") String text = token.getText();\r
 425         int length = text.length();\r
 426 \r
 427         for(int i = length - 1; i >= 0; i--)\r
 428         {\r
 429             this.eof = false;\r
 430 \r
 431             this.in.unread(text.charAt(i));\r
 432         }\r
 433 \r
 434         this.pos = token.getPos() - 1;\r
 435         this.line = token.getLine() - 1;\r
 436     }\r
 437 \r
 438     private String getText(int acceptLength)\r
 439     {\r
 440         StringBuffer s = new StringBuffer(acceptLength);\r
 441         for(int i = 0; i < acceptLength; i++)\r
 442         {\r
 443             s.append(this.text.charAt(i));\r
 444         }\r
 445 \r
 446         return s.toString();\r
 447     }\r
 448 \r
 449     private static int[][][][] gotoTable;\r
 450 /*  {\r
 451         { // INITIAL\r
 452             {{10, 10, 1}, {13, 13, 2}, {32, 32, 3}, {34, 34, 4}, {38, 38, 5}, {39, 39, 6}, {40, 40, 7}, {41, 41, 8}, {42, 42, 9}, {43, 43, 10}, {44, 44, 11}, {45, 45, 12}, {47, 47, 13}, {48, 57, 14}, {58, 58, 15}, {60, 60, 16}, {61, 61, 17}, {62, 62, 18}, {65, 90, 19}, {95, 95, 20}, {97, 122, 19}, },\r
 453             {},\r
 454             {{10, 10, 21}, },\r
 455             {{32, 32, 3}, },\r
 456             {{32, 33, 22}, {34, 34, 23}, {35, 38, 22}, {39, 39, 24}, {40, 91, 22}, {93, 127, 22}, },\r
 457             {},\r
 458             {{32, 33, 25}, {34, 34, 26}, {35, 38, 25}, {39, 39, 27}, {40, 91, 25}, {93, 127, 25}, },\r
 459             {},\r
 460             {},\r
 461             {},\r
 462             {},\r
 463             {},\r
 464             {},\r
 465             {},\r
 466             {{46, 46, 28}, {48, 57, 29}, },\r
 467             {},\r
 468             {},\r
 469             {},\r
 470             {},\r
 471             {{40, 40, 30}, {48, 57, 31}, {65, 90, 32}, {97, 122, 32}, },\r
 472             {{65, 90, 33}, {97, 122, 33}, },\r
 473             {},\r
 474             {{32, 127, -6}, },\r
 475             {},\r
 476             {{32, 127, -6}, },\r
 477             {{32, 127, -8}, },\r
 478             {{32, 127, -8}, },\r
 479             {},\r
 480             {{46, 46, 28}, {48, 57, 34}, },\r
 481             {{46, 57, -16}, },\r
 482             {},\r
 483             {{40, 57, -21}, {65, 90, 35}, {97, 122, 35}, },\r
 484             {{40, 122, -21}, },\r
 485             {{48, 57, 36}, {65, 122, -22}, },\r
 486             {{48, 57, 34}, },\r
 487             {{40, 40, 30}, {48, 57, 37}, {65, 122, -33}, },\r
 488             {{48, 57, 36}, },\r
 489             {{40, 122, -37}, },\r
 490         }\r
 491     };*/\r
 492 \r
 493     private static int[][] accept;\r
 494 /*  {\r
 495         // INITIAL\r
 496         {17, 16, 16, 17, -1, 7, -1, 10, 11, 5, 3, 8, 4, 6, 14, 9, 0, 2, 1, -1, -1, 16, -1, 15, -1, -1, -1, 15, 14, 14, 12, 13, -1, -1, 14, -1, 13, -1, },\r
 497 \r
 498     };*/\r
 499 \r
 500     public static class State\r
 501     {\r
 502         public final static State INITIAL = new State(0);\r
 503 \r
 504         private int id;\r
 505 \r
 506         private State(@SuppressWarnings("hiding") int id)\r
 507         {\r
 508             this.id = id;\r
 509         }\r
 510 \r
 511         public int id()\r
 512         {\r
 513             return this.id;\r
 514         }\r
 515     }\r
 516 \r
 517     static \r
 518     {\r
 519         try\r
 520         {\r
 521             DataInputStream s = new DataInputStream(\r
 522                 new BufferedInputStream(\r
 523                 Lexer.class.getResourceAsStream("lexer.dat")));\r
 524 \r
 525             // read gotoTable\r
 526             int length = s.readInt();\r
 527             gotoTable = new int[length][][][];\r
 528             for(int i = 0; i < gotoTable.length; i++)\r
 529             {\r
 530                 length = s.readInt();\r
 531                 gotoTable[i] = new int[length][][];\r
 532                 for(int j = 0; j < gotoTable[i].length; j++)\r
 533                 {\r
 534                     length = s.readInt();\r
 535                     gotoTable[i][j] = new int[length][3];\r
 536                     for(int k = 0; k < gotoTable[i][j].length; k++)\r
 537                     {\r
 538                         for(int l = 0; l < 3; l++)\r
 539                         {\r
 540                             gotoTable[i][j][k][l] = s.readInt();\r
 541                         }\r
 542                     }\r
 543                 }\r
 544             }\r
 545 \r
 546             // read accept\r
 547             length = s.readInt();\r
 548             accept = new int[length][];\r
 549             for(int i = 0; i < accept.length; i++)\r
 550             {\r
 551                 length = s.readInt();\r
 552                 accept[i] = new int[length];\r
 553                 for(int j = 0; j < accept[i].length; j++)\r
 554                 {\r
 555                     accept[i][j] = s.readInt();\r
 556                 }\r
 557             }\r
 558 \r
 559             s.close();\r
 560         }\r
 561         catch(Exception e)\r
 562         {\r
 563             throw new RuntimeException("The file \"lexer.dat\" is either missing or corrupted.");\r
 564         }\r
 565     }\r
 566 }\r