bundles/org.simantics.databoard/cpp/DataBoardTest/libantlr3c-3.2/src/antlr3lexer.c

   1 /** \file\r
   2  *\r
   3  * Base implementation of an antlr 3 lexer.\r
   4  *\r
   5  * An ANTLR3 lexer implements a base recongizer, a token source and\r
   6  * a lexer interface. It constructs a base recognizer with default\r
   7  * functions, then overrides any of these that are parser specific (usual\r
   8  * default implementation of base recognizer.\r
   9  */\r
  10 \r
  11 // [The "BSD licence"]\r
  12 // Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC\r
  13 // http://www.temporal-wave.com\r
  14 // http://www.linkedin.com/in/jimidle\r
  15 //\r
  16 // All rights reserved.\r
  17 //\r
  18 // Redistribution and use in source and binary forms, with or without\r
  19 // modification, are permitted provided that the following conditions\r
  20 // are met:\r
  21 // 1. Redistributions of source code must retain the above copyright\r
  22 //    notice, this list of conditions and the following disclaimer.\r
  23 // 2. Redistributions in binary form must reproduce the above copyright\r
  24 //    notice, this list of conditions and the following disclaimer in the\r
  25 //    documentation and/or other materials provided with the distribution.\r
  26 // 3. The name of the author may not be used to endorse or promote products\r
  27 //    derived from this software without specific prior written permission.\r
  28 //\r
  29 // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\r
  30 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\r
  31 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\r
  32 // IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\r
  33 // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\r
  34 // NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\r
  35 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\r
  36 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r
  37 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\r
  38 // THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r
  39 \r
  40 #include    <antlr3lexer.h>\r
  41 \r
  42 static void                                     mTokens                                         (pANTLR3_LEXER lexer);\r
  43 static void                                     setCharStream                           (pANTLR3_LEXER lexer,  pANTLR3_INPUT_STREAM input);\r
  44 static void                                     pushCharStream                          (pANTLR3_LEXER lexer,  pANTLR3_INPUT_STREAM input);\r
  45 static void                                     popCharStream                           (pANTLR3_LEXER lexer);\r
  46 \r
  47 static void                                     emitNew                                         (pANTLR3_LEXER lexer,  pANTLR3_COMMON_TOKEN token);\r
  48 static pANTLR3_COMMON_TOKEN emit                                                (pANTLR3_LEXER lexer);\r
  49 static ANTLR3_BOOLEAN       matchs                                              (pANTLR3_LEXER lexer, ANTLR3_UCHAR * string);\r
  50 static ANTLR3_BOOLEAN       matchc                                              (pANTLR3_LEXER lexer, ANTLR3_UCHAR c);\r
  51 static ANTLR3_BOOLEAN       matchRange                                  (pANTLR3_LEXER lexer, ANTLR3_UCHAR low, ANTLR3_UCHAR high);\r
  52 static void                                     matchAny                                        (pANTLR3_LEXER lexer);\r
  53 static void                                     recover                                         (pANTLR3_LEXER lexer);\r
  54 static ANTLR3_UINT32        getLine                                             (pANTLR3_LEXER lexer);\r
  55 static ANTLR3_MARKER        getCharIndex                                (pANTLR3_LEXER lexer);\r
  56 static ANTLR3_UINT32        getCharPositionInLine               (pANTLR3_LEXER lexer);\r
  57 static pANTLR3_STRING       getText                                             (pANTLR3_LEXER lexer);\r
  58 static pANTLR3_COMMON_TOKEN nextToken                                   (pANTLR3_TOKEN_SOURCE toksource);\r
  59 \r
  60 static void                                     displayRecognitionError     (pANTLR3_BASE_RECOGNIZER rec, pANTLR3_UINT8 * tokenNames);\r
  61 static void                                     reportError                                     (pANTLR3_BASE_RECOGNIZER rec);\r
  62 static void *                           getCurrentInputSymbol           (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM istream);\r
  63 static void *                           getMissingSymbol                        (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM istream, pANTLR3_EXCEPTION      e,\r
  64                                                                                                                         ANTLR3_UINT32 expectedTokenType, pANTLR3_BITSET_LIST follow);\r
  65 \r
  66 static void                                     reset                                           (pANTLR3_BASE_RECOGNIZER rec);\r
  67 \r
  68 static void                                     freeLexer                                       (pANTLR3_LEXER lexer);\r
  69 \r
  70 \r
  71 ANTLR3_API pANTLR3_LEXER\r
  72 antlr3LexerNew(ANTLR3_UINT32 sizeHint, pANTLR3_RECOGNIZER_SHARED_STATE state)\r
  73 {\r
  74     pANTLR3_LEXER   lexer;\r
  75     pANTLR3_COMMON_TOKEN        specialT;\r
  76 \r
  77         /* Allocate memory\r
  78         */\r
  79         lexer   = (pANTLR3_LEXER) ANTLR3_MALLOC(sizeof(ANTLR3_LEXER));\r
  80 \r
  81         if      (lexer == NULL)\r
  82         {\r
  83                 return  NULL;\r
  84         }\r
  85 \r
  86         /* Now we need to create the base recognizer\r
  87         */\r
  88         lexer->rec          =  antlr3BaseRecognizerNew(ANTLR3_TYPE_LEXER, sizeHint, state);\r
  89 \r
  90         if      (lexer->rec == NULL)\r
  91         {\r
  92                 lexer->free(lexer);\r
  93                 return  NULL;\r
  94         }\r
  95         lexer->rec->super  =  lexer;\r
  96 \r
  97         lexer->rec->displayRecognitionError         = displayRecognitionError;\r
  98         lexer->rec->reportError                                 = reportError;\r
  99         lexer->rec->reset                                               = reset;\r
 100         lexer->rec->getCurrentInputSymbol               = getCurrentInputSymbol;\r
 101         lexer->rec->getMissingSymbol                    = getMissingSymbol;\r
 102 \r
 103         /* Now install the token source interface\r
 104         */\r
 105         if      (lexer->rec->state->tokSource == NULL) \r
 106         {\r
 107                 lexer->rec->state->tokSource    = (pANTLR3_TOKEN_SOURCE)ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_SOURCE));\r
 108 \r
 109                 if      (lexer->rec->state->tokSource == NULL) \r
 110                 {\r
 111                         lexer->rec->free(lexer->rec);\r
 112                         lexer->free(lexer);\r
 113 \r
 114                         return  NULL;\r
 115                 }\r
 116                 lexer->rec->state->tokSource->super    =  lexer;\r
 117 \r
 118                 /* Install the default nextToken() method, which may be overridden\r
 119                  * by generated code, or by anything else in fact.\r
 120                  */\r
 121                 lexer->rec->state->tokSource->nextToken     =  nextToken;\r
 122                 lexer->rec->state->tokSource->strFactory    = NULL;\r
 123 \r
 124                 lexer->rec->state->tokFactory                           = NULL;\r
 125         }\r
 126 \r
 127     /* Install the lexer API\r
 128      */\r
 129     lexer->setCharStream                        =  setCharStream;\r
 130     lexer->mTokens                                      = (void (*)(void *))(mTokens);\r
 131     lexer->setCharStream                        =  setCharStream;\r
 132     lexer->pushCharStream                       =  pushCharStream;\r
 133     lexer->popCharStream                        =  popCharStream;\r
 134     lexer->emit                                         =  emit;\r
 135     lexer->emitNew                                      =  emitNew;\r
 136     lexer->matchs                                       =  matchs;\r
 137     lexer->matchc                                       =  matchc;\r
 138     lexer->matchRange                           =  matchRange;\r
 139     lexer->matchAny                                     =  matchAny;\r
 140     lexer->recover                                      =  recover;\r
 141     lexer->getLine                                      =  getLine;\r
 142     lexer->getCharIndex                         =  getCharIndex;\r
 143     lexer->getCharPositionInLine    =  getCharPositionInLine;\r
 144     lexer->getText                                      =  getText;\r
 145     lexer->free                                         =  freeLexer;\r
 146     \r
 147     /* Initialise the eof token\r
 148      */\r
 149     specialT                            = &(lexer->rec->state->tokSource->eofToken);\r
 150     antlr3SetTokenAPI     (specialT);\r
 151     specialT->setType     (specialT, ANTLR3_TOKEN_EOF);\r
 152     specialT->factoryMade       = ANTLR3_TRUE;                                  // Prevent things trying to free() it\r
 153     specialT->strFactory    = NULL;\r
 154 \r
 155         // Initialize the skip token.\r
 156         //\r
 157     specialT                            = &(lexer->rec->state->tokSource->skipToken);\r
 158     antlr3SetTokenAPI     (specialT);\r
 159     specialT->setType     (specialT, ANTLR3_TOKEN_INVALID);\r
 160     specialT->factoryMade       = ANTLR3_TRUE;                                  // Prevent things trying to free() it\r
 161     specialT->strFactory    = NULL;\r
 162     return  lexer;\r
 163 }\r
 164 \r
 165 static void\r
 166 reset   (pANTLR3_BASE_RECOGNIZER rec)\r
 167 {\r
 168     pANTLR3_LEXER   lexer;\r
 169 \r
 170     lexer   = rec->super;\r
 171 \r
 172     lexer->rec->state->token                    = NULL;\r
 173     lexer->rec->state->type                             = ANTLR3_TOKEN_INVALID;\r
 174     lexer->rec->state->channel                  = ANTLR3_TOKEN_DEFAULT_CHANNEL;\r
 175     lexer->rec->state->tokenStartCharIndex              = -1;\r
 176     lexer->rec->state->tokenStartCharPositionInLine = -1;\r
 177     lexer->rec->state->tokenStartLine           = -1;\r
 178 \r
 179     lexer->rec->state->text         = NULL;\r
 180 \r
 181     if (lexer->input != NULL)\r
 182     {\r
 183         lexer->input->istream->seek(lexer->input->istream, 0);\r
 184     }\r
 185 }\r
 186 \r
 187 ///\r
 188 /// \brief\r
 189 /// Returns the next available token from the current input stream.\r
 190 /// \r
 191 /// \param toksource\r
 192 /// Points to the implementation of a token source. The lexer is \r
 193 /// addressed by the super structure pointer.\r
 194 /// \r
 195 /// \returns\r
 196 /// The next token in the current input stream or the EOF token\r
 197 /// if there are no more tokens.\r
 198 /// \r
 199 /// \remarks\r
 200 /// Write remarks for nextToken here.\r
 201 /// \r
 202 /// \see nextToken\r
 203 ///\r
 204 ANTLR3_INLINE static pANTLR3_COMMON_TOKEN\r
 205 nextTokenStr        (pANTLR3_TOKEN_SOURCE toksource)\r
 206 {\r
 207         pANTLR3_LEXER   lexer;\r
 208 \r
 209         lexer   = (pANTLR3_LEXER)(toksource->super);\r
 210 \r
 211         /// Loop until we get a non skipped token or EOF\r
 212         ///\r
 213         for     (;;)\r
 214         {\r
 215                 // Get rid of any previous token (token factory takes care of\r
 216                 // any de-allocation when this token is finally used up.\r
 217                 //\r
 218                 lexer->rec->state->token                    = NULL;\r
 219                 lexer->rec->state->error                    = ANTLR3_FALSE;         // Start out without an exception\r
 220                 lexer->rec->state->failed                   = ANTLR3_FALSE;\r
 221 \r
 222 \r
 223 \r
 224                 // Now call the matching rules and see if we can generate a new token\r
 225                 //\r
 226                 for     (;;)\r
 227                 {\r
 228             // Record the start of the token in our input stream.\r
 229             //\r
 230             lexer->rec->state->channel                                          = ANTLR3_TOKEN_DEFAULT_CHANNEL;\r
 231             lexer->rec->state->tokenStartCharIndex                      = lexer->input->istream->index(lexer->input->istream);\r
 232             lexer->rec->state->tokenStartCharPositionInLine     = lexer->input->getCharPositionInLine(lexer->input);\r
 233                 lexer->rec->state->tokenStartLine                               = lexer->input->getLine(lexer->input);\r
 234             lexer->rec->state->text                                                     = NULL;\r
 235 \r
 236                         if  (lexer->input->istream->_LA(lexer->input->istream, 1) == ANTLR3_CHARSTREAM_EOF)\r
 237                         {\r
 238                                 // Reached the end of the current stream, nothing more to do if this is\r
 239                                 // the last in the stack.\r
 240                                 //\r
 241                                 pANTLR3_COMMON_TOKEN    teof = &(toksource->eofToken);\r
 242 \r
 243                                 teof->setStartIndex (teof, lexer->getCharIndex(lexer));\r
 244                                 teof->setStopIndex  (teof, lexer->getCharIndex(lexer));\r
 245                                 teof->setLine   (teof, lexer->getLine(lexer));\r
 246                                 teof->factoryMade = ANTLR3_TRUE;        // This isn't really manufactured but it stops things from trying to free it\r
 247                                 return  teof;\r
 248                         }\r
 249 \r
 250                         lexer->rec->state->token                = NULL;\r
 251                         lexer->rec->state->error                = ANTLR3_FALSE;     // Start out without an exception\r
 252                         lexer->rec->state->failed               = ANTLR3_FALSE;\r
 253 \r
 254                         // Call the generated lexer, see if it can get a new token together.\r
 255                         //\r
 256                         lexer->mTokens(lexer->ctx);\r
 257 \r
 258                         if  (lexer->rec->state->error  == ANTLR3_TRUE)\r
 259                         {\r
 260                                 // Recognition exception, report it and try to recover.\r
 261                                 //\r
 262                                 lexer->rec->state->failed           = ANTLR3_TRUE;\r
 263                                 lexer->rec->reportError(lexer->rec);\r
 264                                 lexer->recover(lexer); \r
 265                         }\r
 266                         else\r
 267                         {\r
 268                                 if (lexer->rec->state->token == NULL)\r
 269                                 {\r
 270                                         // Emit the real token, which adds it in to the token stream basically\r
 271                                         //\r
 272                                         emit(lexer);\r
 273                                 }\r
 274                                 else if (lexer->rec->state->token ==  &(toksource->skipToken))\r
 275                                 {\r
 276                                         // A real token could have been generated, but "Computer say's naaaaah" and it\r
 277                                         // it is just something we need to skip altogether.\r
 278                                         //\r
 279                                         continue;\r
 280                                 }\r
 281                                 \r
 282                                 // Good token, not skipped, not EOF token\r
 283                                 //\r
 284                                 return  lexer->rec->state->token;\r
 285                         }\r
 286                 }\r
 287         }\r
 288 }\r
 289 \r
 290 /**\r
 291  * \brief\r
 292  * Default implementation of the nextToken() call for a lexer.\r
 293  * \r
 294  * \param toksource\r
 295  * Points to the implementation of a token source. The lexer is \r
 296  * addressed by the super structure pointer.\r
 297  * \r
 298  * \returns\r
 299  * The next token in the current input stream or the EOF token\r
 300  * if there are no more tokens in any input stream in the stack.\r
 301  * \r
 302  * Write detailed description for nextToken here.\r
 303  * \r
 304  * \remarks\r
 305  * Write remarks for nextToken here.\r
 306  * \r
 307  * \see nextTokenStr\r
 308  */\r
 309 static pANTLR3_COMMON_TOKEN\r
 310 nextToken           (pANTLR3_TOKEN_SOURCE toksource)\r
 311 {\r
 312         pANTLR3_COMMON_TOKEN tok;\r
 313 \r
 314         // Find the next token in the current stream\r
 315         //\r
 316         tok = nextTokenStr(toksource);\r
 317 \r
 318         // If we got to the EOF token then switch to the previous\r
 319         // input stream if there were any and just return the\r
 320         // EOF if there are none. We must check the next token\r
 321         // in any outstanding input stream we pop into the active\r
 322         // role to see if it was sitting at EOF after PUSHing the\r
 323         // stream we just consumed, otherwise we will return EOF\r
 324         // on the reinstalled input stream, when in actual fact\r
 325         // there might be more input streams to POP before the\r
 326         // real EOF of the whole logical inptu stream. Hence we\r
 327         // use a while loop here until we find somethign in the stream\r
 328         // that isn't EOF or we reach the actual end of the last input\r
 329         // stream on the stack.\r
 330         //\r
 331         while   (tok->type == ANTLR3_TOKEN_EOF)\r
 332         {\r
 333                 pANTLR3_LEXER   lexer;\r
 334 \r
 335                 lexer   = (pANTLR3_LEXER)(toksource->super);\r
 336 \r
 337                 if  (lexer->rec->state->streams != NULL && lexer->rec->state->streams->size(lexer->rec->state->streams) > 0)\r
 338                 {\r
 339                         // We have another input stream in the stack so we\r
 340                         // need to revert to it, then resume the loop to check\r
 341                         // it wasn't sitting at EOF itself.\r
 342                         //\r
 343                         lexer->popCharStream(lexer);\r
 344                         tok = nextTokenStr(toksource);\r
 345                 }\r
 346                 else\r
 347                 {\r
 348                         // There were no more streams on the input stack\r
 349                         // so this EOF is the 'real' logical EOF for\r
 350                         // the input stream. So we just exit the loop and \r
 351                         // return the EOF we have found.\r
 352                         //\r
 353                         break;\r
 354                 }\r
 355                 \r
 356         }\r
 357 \r
 358         // return whatever token we have, which may be EOF\r
 359         //\r
 360         return  tok;\r
 361 }\r
 362 \r
 363 ANTLR3_API pANTLR3_LEXER\r
 364 antlr3LexerNewStream(ANTLR3_UINT32 sizeHint, pANTLR3_INPUT_STREAM input, pANTLR3_RECOGNIZER_SHARED_STATE state)\r
 365 {\r
 366     pANTLR3_LEXER   lexer;\r
 367 \r
 368     // Create a basic lexer first\r
 369     //\r
 370     lexer   = antlr3LexerNew(sizeHint, state);\r
 371 \r
 372     if  (lexer != NULL) \r
 373     {\r
 374                 // Install the input stream and reset the lexer\r
 375                 //\r
 376                 setCharStream(lexer, input);\r
 377     }\r
 378 \r
 379     return  lexer;\r
 380 }\r
 381 \r
 382 static void mTokens         (pANTLR3_LEXER lexer)\r
 383 {\r
 384     if  (lexer)     // Fool compiler, avoid pragmas\r
 385     {\r
 386                 ANTLR3_FPRINTF(stderr, "lexer->mTokens(): Error: No lexer rules were added to the lexer yet!\n");\r
 387     }\r
 388 }\r
 389 \r
 390 static void                     \r
 391 reportError                 (pANTLR3_BASE_RECOGNIZER rec)\r
 392 {\r
 393     // Indicate this recognizer had an error while processing.\r
 394         //\r
 395         rec->state->errorCount++;\r
 396 \r
 397     rec->displayRecognitionError(rec, rec->state->tokenNames);\r
 398 }\r
 399 \r
 400 #ifdef  ANTLR3_WINDOWS\r
 401 #pragma warning( disable : 4100 )\r
 402 #endif\r
 403 \r
 404 /** Default lexer error handler (works for 8 bit streams only!!!)\r
 405  */\r
 406 static void                     \r
 407 displayRecognitionError     (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_UINT8 * tokenNames)\r
 408 {\r
 409     pANTLR3_LEXER                       lexer;\r
 410         pANTLR3_EXCEPTION           ex;\r
 411         pANTLR3_STRING                  ftext;\r
 412 \r
 413     lexer   = (pANTLR3_LEXER)(recognizer->super);\r
 414         ex              = lexer->rec->state->exception;\r
 415 \r
 416         // See if there is a 'filename' we can use\r
 417     //\r
 418     if  (ex->name == NULL)\r
 419     {\r
 420                 ANTLR3_FPRINTF(stderr, "-unknown source-(");\r
 421     }\r
 422     else\r
 423     {\r
 424                 ftext = ex->streamName->to8(ex->streamName);\r
 425                 ANTLR3_FPRINTF(stderr, "%s(", ftext->chars);\r
 426     }\r
 427 \r
 428     ANTLR3_FPRINTF(stderr, "%d) ", recognizer->state->exception->line);\r
 429     ANTLR3_FPRINTF(stderr, ": lexer error %d :\n\t%s at offset %d, ", \r
 430                                                 ex->type,\r
 431                                                 (pANTLR3_UINT8)    (ex->message),\r
 432                                             ex->charPositionInLine+1\r
 433                     );\r
 434         {\r
 435                 ANTLR3_INT32    width;\r
 436 \r
 437                 width   = ANTLR3_UINT32_CAST(( (pANTLR3_UINT8)(lexer->input->data) + (lexer->input->size(lexer->input) )) - (pANTLR3_UINT8)(ex->index));\r
 438 \r
 439                 if      (width >= 1)\r
 440                 {                       \r
 441                         if      (isprint(ex->c))\r
 442                         {\r
 443                                 ANTLR3_FPRINTF(stderr, "near '%c' :\n", ex->c);\r
 444                         }\r
 445                         else\r
 446                         {\r
 447                                 ANTLR3_FPRINTF(stderr, "near char(%#02X) :\n", (ANTLR3_UINT8)(ex->c));\r
 448                         }\r
 449                         ANTLR3_FPRINTF(stderr, "\t%.*s\n", width > 20 ? 20 : width ,((pANTLR3_UINT8)ex->index));\r
 450                 }\r
 451                 else\r
 452                 {\r
 453                         ANTLR3_FPRINTF(stderr, "(end of input).\n\t This indicates a poorly specified lexer RULE\n\t or unterminated input element such as: \"STRING[\"]\n");\r
 454                         ANTLR3_FPRINTF(stderr, "\t The lexer was matching from line %d, offset %d, which\n\t ", \r
 455                                                                 (ANTLR3_UINT32)(lexer->rec->state->tokenStartLine),\r
 456                                                                 (ANTLR3_UINT32)(lexer->rec->state->tokenStartCharPositionInLine)\r
 457                                                                 );\r
 458                         width = ANTLR3_UINT32_CAST(((pANTLR3_UINT8)(lexer->input->data)+(lexer->input->size(lexer->input))) - (pANTLR3_UINT8)(lexer->rec->state->tokenStartCharIndex));\r
 459 \r
 460                         if      (width >= 1)\r
 461                         {\r
 462                                 ANTLR3_FPRINTF(stderr, "looks like this:\n\t\t%.*s\n", width > 20 ? 20 : width ,(pANTLR3_UINT8)(lexer->rec->state->tokenStartCharIndex));\r
 463                         }\r
 464                         else\r
 465                         {\r
 466                                 ANTLR3_FPRINTF(stderr, "is also the end of the line, so you must check your lexer rules\n");\r
 467                         }\r
 468                 }\r
 469         }\r
 470 }\r
 471 \r
 472 static void setCharStream   (pANTLR3_LEXER lexer,  pANTLR3_INPUT_STREAM input)\r
 473 {\r
 474     /* Install the input interface\r
 475      */\r
 476     lexer->input        = input;\r
 477 \r
 478     /* We may need a token factory for the lexer; we don't destroy any existing factory\r
 479      * until the lexer is destroyed, as people may still be using the tokens it produced.\r
 480      * TODO: Later I will provide a dup() method for a token so that it can extract itself\r
 481      * out of the factory. \r
 482      */\r
 483     if  (lexer->rec->state->tokFactory == NULL)\r
 484     {\r
 485         lexer->rec->state->tokFactory   = antlr3TokenFactoryNew(input);\r
 486     }\r
 487     else\r
 488     {\r
 489         /* When the input stream is being changed on the fly, rather than\r
 490          * at the start of a new lexer, then we must tell the tokenFactory\r
 491          * which input stream to adorn the tokens with so that when they\r
 492          * are asked to provide their original input strings they can\r
 493          * do so from the correct text stream.\r
 494          */\r
 495         lexer->rec->state->tokFactory->setInputStream(lexer->rec->state->tokFactory, input);\r
 496     }\r
 497 \r
 498     /* Propagate the string factory so that we preserve the encoding form from\r
 499      * the input stream.\r
 500      */\r
 501     if  (lexer->rec->state->tokSource->strFactory == NULL)\r
 502     {\r
 503         lexer->rec->state->tokSource->strFactory        = input->strFactory;\r
 504 \r
 505         // Set the newly acquired string factory up for our pre-made tokens\r
 506         // for EOF.\r
 507         //\r
 508         if (lexer->rec->state->tokSource->eofToken.strFactory == NULL)\r
 509         {\r
 510             lexer->rec->state->tokSource->eofToken.strFactory = input->strFactory;\r
 511         }\r
 512     }\r
 513 \r
 514     /* This is a lexer, install the appropriate exception creator\r
 515      */\r
 516     lexer->rec->exConstruct = antlr3RecognitionExceptionNew;\r
 517 \r
 518     /* Set the current token to nothing\r
 519      */\r
 520     lexer->rec->state->token            = NULL;\r
 521     lexer->rec->state->text                     = NULL;\r
 522     lexer->rec->state->tokenStartCharIndex      = -1;\r
 523 \r
 524     /* Copy the name of the char stream to the token source\r
 525      */\r
 526     lexer->rec->state->tokSource->fileName = input->fileName;\r
 527 }\r
 528 \r
 529 /*!\r
 530  * \brief\r
 531  * Change to a new input stream, remembering the old one.\r
 532  * \r
 533  * \param lexer\r
 534  * Pointer to the lexer instance to switch input streams for.\r
 535  * \r
 536  * \param input\r
 537  * New input stream to install as the current one.\r
 538  * \r
 539  * Switches the current character input stream to \r
 540  * a new one, saving the old one, which we will revert to at the end of this \r
 541  * new one.\r
 542  */\r
 543 static void\r
 544 pushCharStream  (pANTLR3_LEXER lexer,  pANTLR3_INPUT_STREAM input)\r
 545 {\r
 546         // Do we need a new input stream stack?\r
 547         //\r
 548         if      (lexer->rec->state->streams == NULL)\r
 549         {\r
 550                 // This is the first call to stack a new\r
 551                 // stream and so we must create the stack first.\r
 552                 //\r
 553                 lexer->rec->state->streams = antlr3StackNew(0);\r
 554 \r
 555                 if  (lexer->rec->state->streams == NULL)\r
 556                 {\r
 557                         // Could not do this, we just fail to push it.\r
 558                         // TODO: Consider if this is what we want to do, but then\r
 559                         //       any programmer can override this method to do something else.\r
 560                         return;\r
 561                 }\r
 562         }\r
 563 \r
 564         // We have a stack, so we can save the current input stream\r
 565         // into it.\r
 566         //\r
 567         lexer->input->istream->mark(lexer->input->istream);\r
 568         lexer->rec->state->streams->push(lexer->rec->state->streams, lexer->input, NULL);\r
 569 \r
 570         // And now we can install this new one\r
 571         //\r
 572         lexer->setCharStream(lexer, input);\r
 573 }\r
 574 \r
 575 /*!\r
 576  * \brief\r
 577  * Stops using the current input stream and reverts to any prior\r
 578  * input stream on the stack.\r
 579  * \r
 580  * \param lexer\r
 581  * Description of parameter lexer.\r
 582  * \r
 583  * Pointer to a function that abandons the current input stream, whether it\r
 584  * is empty or not and reverts to the previous stacked input stream.\r
 585  *\r
 586  * \remark\r
 587  * The function fails silently if there are no prior input streams.\r
 588  */\r
 589 static void\r
 590 popCharStream   (pANTLR3_LEXER lexer)\r
 591 {\r
 592     pANTLR3_INPUT_STREAM input;\r
 593 \r
 594     // If we do not have a stream stack or we are already at the\r
 595     // stack bottom, then do nothing.\r
 596     //\r
 597     if  (lexer->rec->state->streams != NULL && lexer->rec->state->streams->size(lexer->rec->state->streams) > 0)\r
 598     {\r
 599         // We just leave the current stream to its fate, we do not close\r
 600         // it or anything as we do not know what the programmer intended\r
 601         // for it. This method can always be overridden of course.\r
 602         // So just find out what was currently saved on the stack and use\r
 603         // that now, then pop it from the stack.\r
 604         //\r
 605         input   = (pANTLR3_INPUT_STREAM)(lexer->rec->state->streams->top);\r
 606         lexer->rec->state->streams->pop(lexer->rec->state->streams);\r
 607 \r
 608         // Now install the stream as the current one.\r
 609         //\r
 610         lexer->setCharStream(lexer, input);\r
 611         lexer->input->istream->rewindLast(lexer->input->istream);\r
 612     }\r
 613     return;\r
 614 }\r
 615 \r
 616 static void emitNew         (pANTLR3_LEXER lexer,  pANTLR3_COMMON_TOKEN token)\r
 617 {\r
 618     lexer->rec->state->token    = token;        /* Voila!   */\r
 619 }\r
 620 \r
 621 static pANTLR3_COMMON_TOKEN\r
 622 emit        (pANTLR3_LEXER lexer)\r
 623 {\r
 624     pANTLR3_COMMON_TOKEN        token;\r
 625 \r
 626     /* We could check pointers to token factories and so on, but\r
 627      * we are in code that we want to run as fast as possible\r
 628      * so we are not checking any errors. So make sure you have installed an input stream before\r
 629      * trying to emit a new token.\r
 630      */\r
 631     token   = lexer->rec->state->tokFactory->newToken(lexer->rec->state->tokFactory);\r
 632 \r
 633     /* Install the supplied information, and some other bits we already know\r
 634      * get added automatically, such as the input stream it is associated with\r
 635      * (though it can all be overridden of course)\r
 636      */\r
 637     token->type             = lexer->rec->state->type;\r
 638     token->channel          = lexer->rec->state->channel;\r
 639     token->start            = lexer->rec->state->tokenStartCharIndex;\r
 640     token->stop             = lexer->getCharIndex(lexer) - 1;\r
 641     token->line             = lexer->rec->state->tokenStartLine;\r
 642     token->charPosition = lexer->rec->state->tokenStartCharPositionInLine;\r
 643 \r
 644         if      (lexer->rec->state->text != NULL)\r
 645         {\r
 646                 token->textState                = ANTLR3_TEXT_STRING;\r
 647                 token->tokText.text         = lexer->rec->state->text;\r
 648         }\r
 649         else\r
 650         {\r
 651                 token->textState        = ANTLR3_TEXT_NONE;\r
 652         }\r
 653     token->lineStart    = lexer->input->currentLine;\r
 654         token->user1            = lexer->rec->state->user1;\r
 655         token->user2            = lexer->rec->state->user2;\r
 656         token->user3            = lexer->rec->state->user3;\r
 657         token->custom           = lexer->rec->state->custom;\r
 658 \r
 659     lexer->rec->state->token        = token;\r
 660 \r
 661     return  token;\r
 662 }\r
 663 \r
 664 /**\r
 665  * Free the resources allocated by a lexer\r
 666  */\r
 667 static void \r
 668 freeLexer    (pANTLR3_LEXER lexer)\r
 669 {\r
 670         // This may have ben a delegate or delegator lexer, in which case the\r
 671         // state may already have been freed (and set to NULL therefore)\r
 672         // so we ignore the state if we don't have it.\r
 673         //\r
 674         if      (lexer->rec->state != NULL)\r
 675         {\r
 676                 if      (lexer->rec->state->streams != NULL)\r
 677                 {\r
 678                         lexer->rec->state->streams->free(lexer->rec->state->streams);\r
 679                 }\r
 680                 if      (lexer->rec->state->tokFactory != NULL)\r
 681                 {\r
 682                         lexer->rec->state->tokFactory->close(lexer->rec->state->tokFactory);\r
 683                         lexer->rec->state->tokFactory = NULL;\r
 684                 }\r
 685                 if      (lexer->rec->state->tokSource != NULL)\r
 686                 {\r
 687                         ANTLR3_FREE(lexer->rec->state->tokSource);\r
 688                         lexer->rec->state->tokSource = NULL;\r
 689                 }\r
 690         }\r
 691         if      (lexer->rec != NULL)\r
 692         {\r
 693                 lexer->rec->free(lexer->rec);\r
 694                 lexer->rec = NULL;\r
 695         }\r
 696         ANTLR3_FREE(lexer);\r
 697 }\r
 698 \r
 699 /** Implementation of matchs for the lexer, overrides any\r
 700  *  base implementation in the base recognizer. \r
 701  *\r
 702  *  \remark\r
 703  *  Note that the generated code lays down arrays of ints for constant\r
 704  *  strings so that they are int UTF32 form!\r
 705  */\r
 706 static ANTLR3_BOOLEAN\r
 707 matchs(pANTLR3_LEXER lexer, ANTLR3_UCHAR * string)\r
 708 {\r
 709         while   (*string != ANTLR3_STRING_TERMINATOR)\r
 710         {\r
 711                 if  (lexer->input->istream->_LA(lexer->input->istream, 1) != (*string))\r
 712                 {\r
 713                         if      (lexer->rec->state->backtracking > 0)\r
 714                         {\r
 715                                 lexer->rec->state->failed = ANTLR3_TRUE;\r
 716                                 return ANTLR3_FALSE;\r
 717                         }\r
 718 \r
 719                         lexer->rec->exConstruct(lexer->rec);\r
 720                         lexer->rec->state->failed        = ANTLR3_TRUE;\r
 721 \r
 722                         /* TODO: Implement exception creation more fully perhaps\r
 723                          */\r
 724                         lexer->recover(lexer);\r
 725                         return  ANTLR3_FALSE;\r
 726                 }\r
 727 \r
 728                 /* Matched correctly, do consume it\r
 729                  */\r
 730                 lexer->input->istream->consume(lexer->input->istream);\r
 731                 string++;\r
 732 \r
 733                 /* Reset any failed indicator\r
 734                  */\r
 735                 lexer->rec->state->failed = ANTLR3_FALSE;\r
 736         }\r
 737 \r
 738 \r
 739         return  ANTLR3_TRUE;\r
 740 }\r
 741 \r
 742 /** Implementation of matchc for the lexer, overrides any\r
 743  *  base implementation in the base recognizer. \r
 744  *\r
 745  *  \remark\r
 746  *  Note that the generated code lays down arrays of ints for constant\r
 747  *  strings so that they are int UTF32 form!\r
 748  */\r
 749 static ANTLR3_BOOLEAN\r
 750 matchc(pANTLR3_LEXER lexer, ANTLR3_UCHAR c)\r
 751 {\r
 752         if      (lexer->input->istream->_LA(lexer->input->istream, 1) == c)\r
 753         {\r
 754                 /* Matched correctly, do consume it\r
 755                  */\r
 756                 lexer->input->istream->consume(lexer->input->istream);\r
 757 \r
 758                 /* Reset any failed indicator\r
 759                  */\r
 760                 lexer->rec->state->failed = ANTLR3_FALSE;\r
 761 \r
 762                 return  ANTLR3_TRUE;\r
 763         }\r
 764 \r
 765         /* Failed to match, exception and recovery time.\r
 766          */\r
 767         if      (lexer->rec->state->backtracking > 0)\r
 768         {\r
 769                 lexer->rec->state->failed  = ANTLR3_TRUE;\r
 770                 return  ANTLR3_FALSE;\r
 771         }\r
 772 \r
 773         lexer->rec->exConstruct(lexer->rec);\r
 774 \r
 775         /* TODO: Implement exception creation more fully perhaps\r
 776          */\r
 777         lexer->recover(lexer);\r
 778 \r
 779         return  ANTLR3_FALSE;\r
 780 }\r
 781 \r
 782 /** Implementation of match range for the lexer, overrides any\r
 783  *  base implementation in the base recognizer. \r
 784  *\r
 785  *  \remark\r
 786  *  Note that the generated code lays down arrays of ints for constant\r
 787  *  strings so that they are int UTF32 form!\r
 788  */\r
 789 static ANTLR3_BOOLEAN\r
 790 matchRange(pANTLR3_LEXER lexer, ANTLR3_UCHAR low, ANTLR3_UCHAR high)\r
 791 {\r
 792     ANTLR3_UCHAR    c;\r
 793 \r
 794     /* What is in the stream at the moment?\r
 795      */\r
 796     c   = lexer->input->istream->_LA(lexer->input->istream, 1);\r
 797     if  ( c >= low && c <= high)\r
 798     {\r
 799         /* Matched correctly, consume it\r
 800          */\r
 801         lexer->input->istream->consume(lexer->input->istream);\r
 802 \r
 803         /* Reset any failed indicator\r
 804          */\r
 805         lexer->rec->state->failed = ANTLR3_FALSE;\r
 806 \r
 807         return  ANTLR3_TRUE;\r
 808     }\r
 809     \r
 810     /* Failed to match, execption and recovery time.\r
 811      */\r
 812 \r
 813     if  (lexer->rec->state->backtracking > 0)\r
 814     {\r
 815         lexer->rec->state->failed  = ANTLR3_TRUE;\r
 816         return  ANTLR3_FALSE;\r
 817     }\r
 818 \r
 819     lexer->rec->exConstruct(lexer->rec);\r
 820 \r
 821     /* TODO: Implement exception creation more fully\r
 822      */\r
 823     lexer->recover(lexer);\r
 824 \r
 825     return  ANTLR3_FALSE;\r
 826 }\r
 827 \r
 828 static void\r
 829 matchAny            (pANTLR3_LEXER lexer)\r
 830 {\r
 831     lexer->input->istream->consume(lexer->input->istream);\r
 832 }\r
 833 \r
 834 static void\r
 835 recover     (pANTLR3_LEXER lexer)\r
 836 {\r
 837     lexer->input->istream->consume(lexer->input->istream);\r
 838 }\r
 839 \r
 840 static ANTLR3_UINT32\r
 841 getLine     (pANTLR3_LEXER lexer)\r
 842 {\r
 843     return  lexer->input->getLine(lexer->input);\r
 844 }\r
 845 \r
 846 static ANTLR3_UINT32\r
 847 getCharPositionInLine   (pANTLR3_LEXER lexer)\r
 848 {\r
 849     return  lexer->input->getCharPositionInLine(lexer->input);\r
 850 }\r
 851 \r
 852 static ANTLR3_MARKER    getCharIndex        (pANTLR3_LEXER lexer)\r
 853 {\r
 854     return lexer->input->istream->index(lexer->input->istream);\r
 855 }\r
 856 \r
 857 static pANTLR3_STRING\r
 858 getText     (pANTLR3_LEXER lexer)\r
 859 {\r
 860         if (lexer->rec->state->text)\r
 861         {\r
 862                 return  lexer->rec->state->text;\r
 863 \r
 864         }\r
 865         return  lexer->input->substr(\r
 866                                                                         lexer->input, \r
 867                                                                         lexer->rec->state->tokenStartCharIndex,\r
 868                                                                         lexer->getCharIndex(lexer) - lexer->input->charByteSize\r
 869                                                         );\r
 870 \r
 871 }\r
 872 \r
 873 static void *                           \r
 874 getCurrentInputSymbol           (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM istream)\r
 875 {\r
 876         return NULL;\r
 877 }\r
 878 \r
 879 static void *                           \r
 880 getMissingSymbol                        (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM istream, pANTLR3_EXCEPTION      e,\r
 881                                                                         ANTLR3_UINT32 expectedTokenType, pANTLR3_BITSET_LIST follow)\r
 882 {\r
 883         return NULL;\r
 884 }\r