bundles/org.simantics.databoard/cpp/DataBoardTest/libantlr3c-3.2/src/antlr3tokenstream.c

   1 /// \file \r
   2 /// Default implementation of CommonTokenStream\r
   3 ///\r
   4 \r
   5 // [The "BSD licence"]\r
   6 // Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC\r
   7 // http://www.temporal-wave.com\r
   8 // http://www.linkedin.com/in/jimidle\r
   9 //\r
  10 // All rights reserved.\r
  11 //\r
  12 // Redistribution and use in source and binary forms, with or without\r
  13 // modification, are permitted provided that the following conditions\r
  14 // are met:\r
  15 // 1. Redistributions of source code must retain the above copyright\r
  16 //    notice, this list of conditions and the following disclaimer.\r
  17 // 2. Redistributions in binary form must reproduce the above copyright\r
  18 //    notice, this list of conditions and the following disclaimer in the\r
  19 //    documentation and/or other materials provided with the distribution.\r
  20 // 3. The name of the author may not be used to endorse or promote products\r
  21 //    derived from this software without specific prior written permission.\r
  22 //\r
  23 // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\r
  24 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\r
  25 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\r
  26 // IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\r
  27 // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\r
  28 // NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\r
  29 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\r
  30 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r
  31 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\r
  32 // THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r
  33 \r
  34 #include    <antlr3tokenstream.h>\r
  35 \r
  36 #ifdef  ANTLR3_WINDOWS\r
  37 #pragma warning( disable : 4100 )\r
  38 #endif\r
  39 \r
  40 // COMMON_TOKEN_STREAM API\r
  41 //\r
  42 static void                                     setTokenTypeChannel     (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel);\r
  43 static void                                     discardTokenType        (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 ttype);\r
  44 static void                                     discardOffChannel       (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_BOOLEAN discard);\r
  45 static pANTLR3_VECTOR           getTokens                       (pANTLR3_COMMON_TOKEN_STREAM cts);\r
  46 static pANTLR3_LIST                     getTokenRange           (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);\r
  47 static pANTLR3_LIST                     getTokensSet            (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types);\r
  48 static pANTLR3_LIST                     getTokensList           (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list);\r
  49 static pANTLR3_LIST                     getTokensType           (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type);\r
  50 \r
  51 // TOKEN_STREAM API \r
  52 //\r
  53 static pANTLR3_COMMON_TOKEN tokLT                               (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);\r
  54 static pANTLR3_COMMON_TOKEN dbgTokLT                    (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);\r
  55 static pANTLR3_COMMON_TOKEN get                                 (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i);\r
  56 static pANTLR3_TOKEN_SOURCE getTokenSource              (pANTLR3_TOKEN_STREAM ts);\r
  57 static void                                     setTokenSource          (pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource);\r
  58 static pANTLR3_STRING       toString                    (pANTLR3_TOKEN_STREAM ts);\r
  59 static pANTLR3_STRING       toStringSS                  (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);\r
  60 static pANTLR3_STRING       toStringTT                  (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop);\r
  61 static void                                     setDebugListener        (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger);\r
  62 \r
  63 // INT STREAM API\r
  64 //\r
  65 static void                                     consume                                         (pANTLR3_INT_STREAM is);\r
  66 static void                                     dbgConsume                                      (pANTLR3_INT_STREAM is);\r
  67 static ANTLR3_UINT32        _LA                                                 (pANTLR3_INT_STREAM is, ANTLR3_INT32 i);\r
  68 static ANTLR3_UINT32        dbgLA                                               (pANTLR3_INT_STREAM is, ANTLR3_INT32 i);\r
  69 static ANTLR3_MARKER        mark                                                (pANTLR3_INT_STREAM is);\r
  70 static ANTLR3_MARKER        dbgMark                                             (pANTLR3_INT_STREAM is);\r
  71 static void                                     release                                         (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark);\r
  72 static ANTLR3_UINT32        size                                                (pANTLR3_INT_STREAM is);\r
  73 static ANTLR3_MARKER            tindex                                          (pANTLR3_INT_STREAM is);\r
  74 static void                                     rewindStream                            (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);\r
  75 static void                                     dbgRewindStream                         (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);\r
  76 static void                                     rewindLast                                      (pANTLR3_INT_STREAM is);\r
  77 static void                                     dbgRewindLast                           (pANTLR3_INT_STREAM is);\r
  78 static void                                     seek                                            (pANTLR3_INT_STREAM is, ANTLR3_MARKER index);\r
  79 static void                                     dbgSeek                                         (pANTLR3_INT_STREAM is, ANTLR3_MARKER index);\r
  80 static pANTLR3_STRING           getSourceName                           (pANTLR3_INT_STREAM is);\r
  81 static void                                     antlr3TokenStreamFree           (pANTLR3_TOKEN_STREAM       stream);\r
  82 static void                                     antlr3CTSFree                           (pANTLR3_COMMON_TOKEN_STREAM    stream);\r
  83 \r
  84 // Helpers\r
  85 //\r
  86 static void                                     fillBuffer                                      (pANTLR3_COMMON_TOKEN_STREAM tokenStream);\r
  87 static ANTLR3_UINT32        skipOffTokenChannels                (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);\r
  88 static ANTLR3_UINT32        skipOffTokenChannelsReverse (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);\r
  89 static pANTLR3_COMMON_TOKEN LB                                                  (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);\r
  90 \r
  91 ANTLR3_API pANTLR3_TOKEN_STREAM\r
  92 antlr3TokenStreamNew()\r
  93 {\r
  94     pANTLR3_TOKEN_STREAM stream;\r
  95 \r
  96     // Memory for the interface structure\r
  97     //\r
  98     stream  = (pANTLR3_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_STREAM));\r
  99 \r
 100     if  (stream == NULL)\r
 101     {\r
 102                 return  NULL;\r
 103     }\r
 104 \r
 105     // Install basic API \r
 106     //\r
 107     stream->free    =  antlr3TokenStreamFree;\r
 108 \r
 109     \r
 110     return stream;\r
 111 }\r
 112 \r
 113 static void\r
 114 antlr3TokenStreamFree(pANTLR3_TOKEN_STREAM stream)\r
 115 {   \r
 116     ANTLR3_FREE(stream);\r
 117 }\r
 118 \r
 119 static void                 \r
 120 antlr3CTSFree       (pANTLR3_COMMON_TOKEN_STREAM stream)\r
 121 {\r
 122         // We only free up our subordinate interfaces if they belong\r
 123         // to us, otherwise we let whoever owns them deal with them.\r
 124         //\r
 125         if      (stream->tstream->super == stream)\r
 126         {\r
 127                 if      (stream->tstream->istream->super == stream->tstream)\r
 128                 {\r
 129                         stream->tstream->istream->free(stream->tstream->istream);\r
 130                         stream->tstream->istream = NULL;\r
 131                 }\r
 132                 stream->tstream->free(stream->tstream);\r
 133         }\r
 134 \r
 135         // Now we free our own resources\r
 136         //\r
 137         if      (stream->tokens != NULL)\r
 138         {\r
 139                 stream->tokens->free(stream->tokens);\r
 140                 stream->tokens  = NULL;\r
 141         }\r
 142         if      (stream->discardSet != NULL)\r
 143         {\r
 144                 stream->discardSet->free(stream->discardSet);\r
 145                 stream->discardSet  = NULL;\r
 146         }\r
 147         if      (stream->channelOverrides != NULL)\r
 148         {\r
 149                 stream->channelOverrides->free(stream->channelOverrides);\r
 150                 stream->channelOverrides = NULL;\r
 151         }\r
 152 \r
 153         // Free our memory now\r
 154         //\r
 155         ANTLR3_FREE(stream);\r
 156 }\r
 157 \r
 158 ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM\r
 159 antlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger)\r
 160 {\r
 161     pANTLR3_COMMON_TOKEN_STREAM stream;\r
 162 \r
 163         // Create a standard token stream\r
 164         //\r
 165         stream = antlr3CommonTokenStreamSourceNew(hint, source);\r
 166 \r
 167         // Install the debugger object\r
 168         //\r
 169         stream->tstream->debugger = debugger;\r
 170 \r
 171         // Override standard token stream methods with debugging versions\r
 172         //\r
 173         stream->tstream->initialStreamState     = ANTLR3_FALSE;\r
 174 \r
 175         stream->tstream->_LT                            = dbgTokLT;\r
 176 \r
 177         stream->tstream->istream->consume               = dbgConsume;\r
 178         stream->tstream->istream->_LA                   = dbgLA;\r
 179         stream->tstream->istream->mark                  = dbgMark;\r
 180         stream->tstream->istream->rewind                = dbgRewindStream;\r
 181         stream->tstream->istream->rewindLast    = dbgRewindLast;\r
 182         stream->tstream->istream->seek                  = dbgSeek;\r
 183 \r
 184         return stream;\r
 185 }\r
 186 \r
 187 ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM\r
 188 antlr3CommonTokenStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source)\r
 189 {\r
 190     pANTLR3_COMMON_TOKEN_STREAM stream;\r
 191 \r
 192     stream = antlr3CommonTokenStreamNew(hint);\r
 193 \r
 194     stream->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;\r
 195     \r
 196     stream->channelOverrides    = NULL;\r
 197     stream->discardSet          = NULL;\r
 198     stream->discardOffChannel   = ANTLR3_FALSE;\r
 199 \r
 200     stream->tstream->setTokenSource(stream->tstream, source);\r
 201 \r
 202     stream->free                =  antlr3CTSFree;\r
 203     return  stream;\r
 204 }\r
 205 \r
 206 ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM\r
 207 antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint)\r
 208 {\r
 209     pANTLR3_COMMON_TOKEN_STREAM stream;\r
 210 \r
 211     /* Memory for the interface structure\r
 212      */\r
 213     stream  = (pANTLR3_COMMON_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TOKEN_STREAM));\r
 214 \r
 215     if  (stream == NULL)\r
 216     {\r
 217         return  NULL;\r
 218     }\r
 219 \r
 220     /* Create space for the token stream interface\r
 221      */\r
 222     stream->tstream         = antlr3TokenStreamNew();\r
 223     stream->tstream->super  =  stream;\r
 224 \r
 225     /* Create space for the INT_STREAM interfacce\r
 226      */\r
 227     stream->tstream->istream                =  antlr3IntStreamNew();\r
 228     stream->tstream->istream->super         =  (stream->tstream);\r
 229     stream->tstream->istream->type          = ANTLR3_TOKENSTREAM;\r
 230 \r
 231     /* Install the token tracking tables\r
 232      */\r
 233     stream->tokens  = antlr3VectorNew(0);\r
 234 \r
 235     /* Defaults\r
 236      */\r
 237     stream->p       = -1;\r
 238 \r
 239     /* Install the common token stream API\r
 240      */\r
 241     stream->setTokenTypeChannel     =  setTokenTypeChannel;\r
 242     stream->discardTokenType        =  discardTokenType;\r
 243     stream->discardOffChannelToks   =  discardOffChannel;\r
 244     stream->getTokens               =  getTokens;\r
 245     stream->getTokenRange           =  getTokenRange;\r
 246     stream->getTokensSet            =  getTokensSet;\r
 247     stream->getTokensList           =  getTokensList;\r
 248     stream->getTokensType           =  getTokensType;\r
 249 \r
 250     /* Install the token stream API\r
 251      */\r
 252     stream->tstream->_LT                                =  tokLT;\r
 253     stream->tstream->get                                =  get;\r
 254     stream->tstream->getTokenSource             =  getTokenSource;\r
 255     stream->tstream->setTokenSource             =  setTokenSource;\r
 256     stream->tstream->toString                   =  toString;\r
 257     stream->tstream->toStringSS                 =  toStringSS;\r
 258     stream->tstream->toStringTT                 =  toStringTT;\r
 259         stream->tstream->setDebugListener       =  setDebugListener;\r
 260 \r
 261     /* Install INT_STREAM interface\r
 262      */\r
 263     stream->tstream->istream->_LA       =  _LA;\r
 264     stream->tstream->istream->mark      =  mark;\r
 265     stream->tstream->istream->release   =  release;\r
 266     stream->tstream->istream->size      =  size;\r
 267     stream->tstream->istream->index     =  tindex;\r
 268     stream->tstream->istream->rewind    =  rewindStream;\r
 269     stream->tstream->istream->rewindLast=  rewindLast;\r
 270     stream->tstream->istream->seek      =  seek;\r
 271     stream->tstream->istream->consume   =  consume;\r
 272         stream->tstream->istream->getSourceName = getSourceName;\r
 273 \r
 274     return  stream;\r
 275 }\r
 276 \r
 277 // Install a debug listener adn switch to debug mode methods\r
 278 //\r
 279 static void                                     \r
 280 setDebugListener        (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger)\r
 281 {\r
 282                 // Install the debugger object\r
 283         //\r
 284         ts->debugger = debugger;\r
 285 \r
 286         // Override standard token stream methods with debugging versions\r
 287         //\r
 288         ts->initialStreamState  = ANTLR3_FALSE;\r
 289 \r
 290         ts->_LT                         = dbgTokLT;\r
 291 \r
 292         ts->istream->consume            = dbgConsume;\r
 293         ts->istream->_LA                        = dbgLA;\r
 294         ts->istream->mark                       = dbgMark;\r
 295         ts->istream->rewind                     = dbgRewindStream;\r
 296         ts->istream->rewindLast         = dbgRewindLast;\r
 297         ts->istream->seek                       = dbgSeek;\r
 298 }\r
 299 \r
 300 /** Get the ith token from the current position 1..n where k=1 is the\r
 301 *  first symbol of lookahead.\r
 302 */\r
 303 static pANTLR3_COMMON_TOKEN \r
 304 tokLT  (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)\r
 305 {\r
 306         ANTLR3_INT32    i;\r
 307         ANTLR3_INT32    n;\r
 308         pANTLR3_COMMON_TOKEN_STREAM cts;\r
 309 \r
 310         cts         = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;\r
 311 \r
 312     if  (k < 0)\r
 313         {\r
 314                 return LB(cts, -k);\r
 315         }\r
 316 \r
 317         if      (cts->p == -1)\r
 318         {\r
 319                 fillBuffer(cts);\r
 320         }\r
 321         if      (k == 0)\r
 322         {\r
 323                 return NULL;\r
 324         }\r
 325 \r
 326         if      ((cts->p + k - 1) >= (ANTLR3_INT32)ts->istream->cachedSize)\r
 327         {\r
 328                 pANTLR3_COMMON_TOKEN    teof = &(ts->tokenSource->eofToken);\r
 329 \r
 330                 teof->setStartIndex (teof, ts->istream->index       (ts->istream));\r
 331                 teof->setStopIndex  (teof, ts->istream->index       (ts->istream));\r
 332                 return  teof;\r
 333         }\r
 334 \r
 335         i       = cts->p;\r
 336         n       = 1;\r
 337 \r
 338         /* Need to find k good tokens, skipping ones that are off channel\r
 339         */\r
 340         while   ( n < k)\r
 341         {\r
 342                 /* Skip off-channel tokens */\r
 343                 i = skipOffTokenChannels(cts, i+1); /* leave p on valid token    */\r
 344                 n++;\r
 345         }\r
 346         if      ( (ANTLR3_UINT32) i >= ts->istream->cachedSize)\r
 347         {\r
 348                 pANTLR3_COMMON_TOKEN    teof = &(ts->tokenSource->eofToken);\r
 349 \r
 350                 teof->setStartIndex (teof, ts->istream->index(ts->istream));\r
 351                 teof->setStopIndex  (teof, ts->istream->index(ts->istream));\r
 352                 return  teof;\r
 353         }\r
 354 \r
 355         // Here the token must be in the input vector. Rather then incut\r
 356         // function call penalty, we jsut return the pointer directly\r
 357         // from the vector\r
 358         //\r
 359         return  (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;\r
 360         //return  (pANTLR3_COMMON_TOKEN)cts->tokens->get(cts->tokens, i);\r
 361 }\r
 362 \r
 363 /// Debug only method to flag consumption of initial off-channel\r
 364 /// tokens in the input stream\r
 365 ///\r
 366 static void\r
 367 consumeInitialHiddenTokens(pANTLR3_INT_STREAM is)\r
 368 {\r
 369         ANTLR3_MARKER   first;\r
 370         ANTLR3_INT32    i;\r
 371         pANTLR3_TOKEN_STREAM    ts;\r
 372 \r
 373         ts          = (pANTLR3_TOKEN_STREAM)        is->super;\r
 374         first   = is->index(is);\r
 375 \r
 376         for     (i=0; i<first; i++)\r
 377         {\r
 378                 ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, i));\r
 379         }\r
 380 \r
 381         ts->initialStreamState = ANTLR3_FALSE;\r
 382 \r
 383 }\r
 384 \r
 385 /// As per the normal tokLT but sends information to the debugger\r
 386 ///\r
 387 static pANTLR3_COMMON_TOKEN \r
 388 dbgTokLT  (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)\r
 389 {\r
 390         if      (ts->initialStreamState == ANTLR3_TRUE)\r
 391         {\r
 392                 consumeInitialHiddenTokens(ts->istream);\r
 393         }\r
 394         return tokLT(ts, k);\r
 395 }\r
 396 \r
 397 #ifdef  ANTLR3_WINDOWS\r
 398         /* When fully optimized VC7 complains about non reachable code.\r
 399          * Not yet sure if this is an optimizer bug, or a bug in the flow analysis\r
 400          */\r
 401 #pragma warning( disable : 4702 )\r
 402 #endif\r
 403 \r
 404 static pANTLR3_COMMON_TOKEN\r
 405 LB(pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 k)\r
 406 {\r
 407     ANTLR3_INT32 i;\r
 408     ANTLR3_INT32 n;\r
 409 \r
 410     if (cts->p == -1)\r
 411     {\r
 412         fillBuffer(cts);\r
 413     }\r
 414     if (k == 0)\r
 415     {\r
 416         return NULL;\r
 417     }\r
 418     if ((cts->p - k) < 0)\r
 419     {\r
 420         return NULL;\r
 421     }\r
 422 \r
 423     i = cts->p;\r
 424     n = 1;\r
 425 \r
 426     /* Need to find k good tokens, going backwards, skipping ones that are off channel\r
 427      */\r
 428     while (n <= (ANTLR3_INT32) k)\r
 429     {\r
 430         /* Skip off-channel tokens\r
 431          */\r
 432 \r
 433         i = skipOffTokenChannelsReverse(cts, i - 1); /* leave p on valid token    */\r
 434         n++;\r
 435     }\r
 436     if (i < 0)\r
 437     {\r
 438         return NULL;\r
 439     }\r
 440         // Here the token must be in the input vector. Rather then incut\r
 441         // function call penalty, we jsut return the pointer directly\r
 442         // from the vector\r
 443         //\r
 444         return  (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;\r
 445 }\r
 446 \r
 447 static pANTLR3_COMMON_TOKEN \r
 448 get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i)\r
 449 {\r
 450     pANTLR3_COMMON_TOKEN_STREAM cts;\r
 451 \r
 452     cts     = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;\r
 453 \r
 454     return  (pANTLR3_COMMON_TOKEN)(cts->tokens->get(cts->tokens, i));  /* Token index is zero based but vectors are 1 based */\r
 455 }\r
 456 \r
 457 static pANTLR3_TOKEN_SOURCE \r
 458 getTokenSource  (pANTLR3_TOKEN_STREAM ts)\r
 459 {\r
 460     return  ts->tokenSource;\r
 461 }\r
 462 \r
 463 static void\r
 464 setTokenSource  (   pANTLR3_TOKEN_STREAM ts,\r
 465                     pANTLR3_TOKEN_SOURCE tokenSource)\r
 466 {\r
 467     ts->tokenSource     = tokenSource;\r
 468 }\r
 469 \r
 470 static pANTLR3_STRING       \r
 471 toString    (pANTLR3_TOKEN_STREAM ts)\r
 472 {\r
 473     pANTLR3_COMMON_TOKEN_STREAM cts;\r
 474 \r
 475     cts     = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;\r
 476 \r
 477     if  (cts->p == -1)\r
 478     {\r
 479         fillBuffer(cts);\r
 480     }\r
 481 \r
 482     return  ts->toStringSS(ts, 0, ts->istream->size(ts->istream));\r
 483 }\r
 484 \r
 485 static pANTLR3_STRING\r
 486 toStringSS(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)\r
 487 {\r
 488     pANTLR3_STRING string;\r
 489     pANTLR3_TOKEN_SOURCE tsource;\r
 490     pANTLR3_COMMON_TOKEN tok;\r
 491     ANTLR3_UINT32 i;\r
 492     pANTLR3_COMMON_TOKEN_STREAM cts;\r
 493 \r
 494     cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;\r
 495 \r
 496     if (cts->p == -1)\r
 497     {\r
 498         fillBuffer(cts);\r
 499     }\r
 500     if (stop >= ts->istream->size(ts->istream))\r
 501     {\r
 502         stop = ts->istream->size(ts->istream) - 1;\r
 503     }\r
 504 \r
 505     /* Who is giving us these tokens?\r
 506      */\r
 507     tsource = ts->getTokenSource(ts);\r
 508 \r
 509     if (tsource != NULL && cts->tokens != NULL)\r
 510     {\r
 511         /* Finally, let's get a string\r
 512          */\r
 513         string = tsource->strFactory->newRaw(tsource->strFactory);\r
 514 \r
 515         for (i = start; i <= stop; i++)\r
 516         {\r
 517             tok = ts->get(ts, i);\r
 518             if (tok != NULL)\r
 519             {\r
 520                 string->appendS(string, tok->getText(tok));\r
 521             }\r
 522         }\r
 523 \r
 524         return string;\r
 525     }\r
 526     return NULL;\r
 527 \r
 528 }\r
 529 \r
 530 static pANTLR3_STRING       \r
 531 toStringTT  (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop)\r
 532 {\r
 533         if      (start != NULL && stop != NULL)\r
 534         {\r
 535                 return  ts->toStringSS(ts, (ANTLR3_UINT32)start->getTokenIndex(start), (ANTLR3_UINT32)stop->getTokenIndex(stop));\r
 536         }\r
 537         else\r
 538         {\r
 539                 return  NULL;\r
 540         }\r
 541 }\r
 542 \r
 543 /** Move the input pointer to the next incoming token.  The stream\r
 544  *  must become active with LT(1) available.  consume() simply\r
 545  *  moves the input pointer so that LT(1) points at the next\r
 546  *  input symbol. Consume at least one token.\r
 547  *\r
 548  *  Walk past any token not on the channel the parser is listening to.\r
 549  */\r
 550 static void                 \r
 551 consume (pANTLR3_INT_STREAM is)\r
 552 {\r
 553         pANTLR3_COMMON_TOKEN_STREAM cts;\r
 554         pANTLR3_TOKEN_STREAM    ts;\r
 555 \r
 556         ts          = (pANTLR3_TOKEN_STREAM)        is->super;\r
 557         cts         = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;\r
 558 \r
 559         if      ((ANTLR3_UINT32)cts->p < cts->tokens->size(cts->tokens))\r
 560         {\r
 561                 cts->p++;\r
 562                 cts->p  = skipOffTokenChannels(cts, cts->p);\r
 563         }\r
 564 }\r
 565 \r
 566 \r
 567 /// As per ordinary consume but notifies the debugger about hidden\r
 568 /// tokens and so on.\r
 569 ///\r
 570 static void\r
 571 dbgConsume      (pANTLR3_INT_STREAM is)\r
 572 {\r
 573         pANTLR3_TOKEN_STREAM    ts;\r
 574         ANTLR3_MARKER                   a;\r
 575         ANTLR3_MARKER                   b;\r
 576         pANTLR3_COMMON_TOKEN    t;\r
 577 \r
 578         ts          = (pANTLR3_TOKEN_STREAM)        is->super;\r
 579 \r
 580         if      (ts->initialStreamState == ANTLR3_TRUE)\r
 581         {\r
 582                 consumeInitialHiddenTokens(is);\r
 583         }\r
 584         \r
 585         a = is->index(is);              // Where are we right now?\r
 586         t = ts->_LT(ts, 1);             // Current token from stream\r
 587 \r
 588         consume(is);                    // Standard consumer\r
 589 \r
 590         b = is->index(is);              // Where are we after consuming 1 on channel token?\r
 591 \r
 592         ts->debugger->consumeToken(ts->debugger, t);    // Tell the debugger that we consumed the first token\r
 593 \r
 594         if      (b>a+1)\r
 595         {\r
 596                 // The standard consume caused the index to advance by more than 1,\r
 597                 // which can only happen if it skipped some off-channel tokens.\r
 598                 // we need to tell the debugger about those tokens.\r
 599                 //\r
 600                 ANTLR3_MARKER   i;\r
 601 \r
 602                 for     (i = a+1; i<b; i++)\r
 603                 {\r
 604                         ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, (ANTLR3_UINT32)i));\r
 605                 }\r
 606 \r
 607         }\r
 608 }\r
 609 \r
 610 /** A simple filter mechanism whereby you can tell this token stream\r
 611  *  to force all tokens of type ttype to be on channel.  For example,\r
 612  *  when interpreting, we cannot execute actions so we need to tell\r
 613  *  the stream to force all WS and NEWLINE to be a different, ignored,\r
 614  *  channel.\r
 615  */\r
 616 static void                 \r
 617 setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel)\r
 618 {\r
 619     if  (tokenStream->channelOverrides == NULL)\r
 620     {\r
 621         tokenStream->channelOverrides   = antlr3ListNew(10);\r
 622     }\r
 623 \r
 624     /* We add one to the channel so we can distinguish NULL as being no entry in the\r
 625      * table for a particular token type.\r
 626      */\r
 627     tokenStream->channelOverrides->put(tokenStream->channelOverrides, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)channel + 1), NULL);\r
 628 }\r
 629 \r
 630 static void                 \r
 631 discardTokenType    (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 ttype)\r
 632 {\r
 633     if  (tokenStream->discardSet == NULL)\r
 634     {\r
 635         tokenStream->discardSet = antlr3ListNew(31);\r
 636     }\r
 637 \r
 638     /* We add one to the channel so we can distinguish NULL as being no entry in the\r
 639      * table for a particular token type. We could use bitsets for this I suppose too.\r
 640      */\r
 641     tokenStream->discardSet->put(tokenStream->discardSet, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)ttype + 1), NULL);\r
 642 }\r
 643 \r
 644 static void                 \r
 645 discardOffChannel   (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_BOOLEAN discard)\r
 646 {\r
 647     tokenStream->discardOffChannel  = discard;\r
 648 }\r
 649 \r
 650 static pANTLR3_VECTOR       \r
 651 getTokens   (pANTLR3_COMMON_TOKEN_STREAM tokenStream)\r
 652 {\r
 653     if  (tokenStream->p == -1)\r
 654     {\r
 655         fillBuffer(tokenStream);\r
 656     }\r
 657 \r
 658     return  tokenStream->tokens;\r
 659 }\r
 660 \r
 661 static pANTLR3_LIST         \r
 662 getTokenRange   (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)\r
 663 {\r
 664     return tokenStream->getTokensSet(tokenStream, start, stop, NULL);\r
 665 }                                                   \r
 666 /** Given a start and stop index, return a List of all tokens in\r
 667  *  the token type BitSet.  Return null if no tokens were found.  This\r
 668  *  method looks at both on and off channel tokens.\r
 669  */\r
 670 static pANTLR3_LIST         \r
 671 getTokensSet    (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types)\r
 672 {\r
 673     pANTLR3_LIST            filteredList;\r
 674     ANTLR3_UINT32           i;\r
 675     ANTLR3_UINT32           n;\r
 676     pANTLR3_COMMON_TOKEN    tok;\r
 677 \r
 678     if  (tokenStream->p == -1)\r
 679     {\r
 680         fillBuffer(tokenStream);\r
 681     }\r
 682     if  (stop > tokenStream->tstream->istream->size(tokenStream->tstream->istream))\r
 683     {\r
 684         stop = tokenStream->tstream->istream->size(tokenStream->tstream->istream);\r
 685     }\r
 686     if  (start > stop)\r
 687     {\r
 688         return NULL;\r
 689     }\r
 690 \r
 691     /* We have the range set, now we need to iterate through the\r
 692      * installed tokens and create a new list with just the ones we want\r
 693      * in it. We are just moving pointers about really.\r
 694      */\r
 695     filteredList    = antlr3ListNew((ANTLR3_UINT32)tokenStream->tstream->istream->size(tokenStream->tstream->istream));\r
 696 \r
 697     for (i = start, n = 0; i<= stop; i++)\r
 698     {\r
 699         tok = tokenStream->tstream->get(tokenStream->tstream, i);\r
 700 \r
 701         if  (      types == NULL\r
 702                 || types->isMember(types, tok->getType(tok) == ANTLR3_TRUE)\r
 703             )\r
 704         {\r
 705             filteredList->put(filteredList, n++, (void *)tok, NULL);\r
 706         }\r
 707     }\r
 708     \r
 709     /* Did we get any then?\r
 710      */\r
 711     if  (filteredList->size(filteredList) == 0)\r
 712     {\r
 713         filteredList->free(filteredList);\r
 714         filteredList    = NULL;\r
 715     }\r
 716 \r
 717     return  filteredList;\r
 718 }\r
 719 \r
 720 static pANTLR3_LIST         \r
 721 getTokensList   (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list)\r
 722 {\r
 723     pANTLR3_BITSET  bitSet;\r
 724     pANTLR3_LIST    newlist;\r
 725 \r
 726     bitSet  = antlr3BitsetList(list->table);\r
 727 \r
 728     newlist    = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);\r
 729 \r
 730     bitSet->free(bitSet);\r
 731 \r
 732     return  newlist;\r
 733 \r
 734 }\r
 735 \r
 736 static pANTLR3_LIST         \r
 737 getTokensType   (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type)\r
 738 {\r
 739     pANTLR3_BITSET  bitSet;\r
 740     pANTLR3_LIST    newlist;\r
 741 \r
 742     bitSet  = antlr3BitsetOf(type, -1);\r
 743     newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);\r
 744 \r
 745     bitSet->free(bitSet);\r
 746 \r
 747     return  newlist;\r
 748 }\r
 749 \r
 750 static ANTLR3_UINT32        \r
 751 _LA  (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)\r
 752 {\r
 753         pANTLR3_TOKEN_STREAM    ts;\r
 754         pANTLR3_COMMON_TOKEN    tok;\r
 755 \r
 756         ts          = (pANTLR3_TOKEN_STREAM)        is->super;\r
 757 \r
 758         tok         =  ts->_LT(ts, i);\r
 759 \r
 760         if      (tok != NULL)\r
 761         {\r
 762                 return  tok->getType(tok);\r
 763         }\r
 764         else\r
 765         {\r
 766                 return  ANTLR3_TOKEN_INVALID;\r
 767         }\r
 768 }\r
 769 \r
 770 /// As per _LA() but for debug mode.\r
 771 ///\r
 772 static ANTLR3_UINT32        \r
 773 dbgLA  (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)\r
 774 {\r
 775     pANTLR3_TOKEN_STREAM    ts;\r
 776    \r
 777     ts      = (pANTLR3_TOKEN_STREAM)        is->super;\r
 778 \r
 779         if      (ts->initialStreamState == ANTLR3_TRUE)\r
 780         {\r
 781                 consumeInitialHiddenTokens(is);\r
 782         }\r
 783         ts->debugger->LT(ts->debugger, i, tokLT(ts, i));\r
 784         return  _LA(is, i);\r
 785 }\r
 786 \r
 787 static ANTLR3_MARKER\r
 788 mark    (pANTLR3_INT_STREAM is)\r
 789 {\r
 790     is->lastMarker = is->index(is);\r
 791     return  is->lastMarker;\r
 792 }\r
 793 \r
 794 /// As per mark() but with a call to tell the debugger we are doing this\r
 795 ///\r
 796 static ANTLR3_MARKER\r
 797 dbgMark (pANTLR3_INT_STREAM is)\r
 798 {\r
 799     pANTLR3_TOKEN_STREAM    ts;\r
 800    \r
 801     ts      = (pANTLR3_TOKEN_STREAM)        is->super;\r
 802         \r
 803         is->lastMarker = is->index(is);\r
 804         ts->debugger->mark(ts->debugger, is->lastMarker);\r
 805 \r
 806     return  is->lastMarker;\r
 807 }\r
 808 \r
 809 static void                 \r
 810 release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark)\r
 811 {\r
 812     return;\r
 813 }\r
 814 \r
 815 static ANTLR3_UINT32        \r
 816 size    (pANTLR3_INT_STREAM is)\r
 817 {\r
 818     pANTLR3_COMMON_TOKEN_STREAM cts;\r
 819     pANTLR3_TOKEN_STREAM        ts;\r
 820 \r
 821     if (is->cachedSize > 0)\r
 822     {\r
 823         return  is->cachedSize;\r
 824     }\r
 825     ts      = (pANTLR3_TOKEN_STREAM)        is->super;\r
 826     cts     = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;\r
 827 \r
 828     is->cachedSize =  cts->tokens->count;\r
 829     return  is->cachedSize;\r
 830 }\r
 831 \r
 832 static ANTLR3_MARKER   \r
 833 tindex  (pANTLR3_INT_STREAM is)\r
 834 {\r
 835     pANTLR3_COMMON_TOKEN_STREAM cts;\r
 836     pANTLR3_TOKEN_STREAM        ts;\r
 837 \r
 838     ts      = (pANTLR3_TOKEN_STREAM)        is->super;\r
 839     cts     = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;\r
 840 \r
 841     return  cts->p;\r
 842 }\r
 843 \r
 844 static void                 \r
 845 dbgRewindLast   (pANTLR3_INT_STREAM is)\r
 846 {\r
 847         pANTLR3_TOKEN_STREAM    ts;\r
 848 \r
 849     ts      = (pANTLR3_TOKEN_STREAM)        is->super;\r
 850 \r
 851         ts->debugger->rewindLast(ts->debugger);\r
 852 \r
 853     is->rewind(is, is->lastMarker);\r
 854 }\r
 855 static void                 \r
 856 rewindLast      (pANTLR3_INT_STREAM is)\r
 857 {\r
 858     is->rewind(is, is->lastMarker);\r
 859 }\r
 860 static void                 \r
 861 rewindStream    (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)\r
 862 {\r
 863     is->seek(is, (ANTLR3_UINT32)(marker));\r
 864 }\r
 865 static void                 \r
 866 dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)\r
 867 {\r
 868     pANTLR3_TOKEN_STREAM        ts;\r
 869 \r
 870     ts      = (pANTLR3_TOKEN_STREAM)        is->super;\r
 871 \r
 872         ts->debugger->rewind(ts->debugger, marker);\r
 873 \r
 874     is->seek(is, (ANTLR3_UINT32)(marker));\r
 875 }\r
 876 \r
 877 static void                 \r
 878 seek    (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)\r
 879 {\r
 880     pANTLR3_COMMON_TOKEN_STREAM cts;\r
 881     pANTLR3_TOKEN_STREAM        ts;\r
 882 \r
 883     ts      = (pANTLR3_TOKEN_STREAM)        is->super;\r
 884     cts     = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;\r
 885 \r
 886     cts->p  = (ANTLR3_UINT32)index;\r
 887 }\r
 888 static void                 \r
 889 dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)\r
 890 {\r
 891         // TODO: Implement seek in debugger when Ter adds it to Java\r
 892         //\r
 893         seek(is, index);\r
 894 }\r
 895 ANTLR3_API void\r
 896 fillBufferExt(pANTLR3_COMMON_TOKEN_STREAM tokenStream)\r
 897 {\r
 898     fillBuffer(tokenStream);\r
 899 }\r
 900 static void\r
 901 fillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream) {\r
 902     ANTLR3_UINT32 index;\r
 903     pANTLR3_COMMON_TOKEN tok;\r
 904     ANTLR3_BOOLEAN discard;\r
 905     void * channelI;\r
 906 \r
 907     /* Start at index 0 of course\r
 908      */\r
 909     index = 0;\r
 910 \r
 911     /* Pick out the next token from the token source\r
 912      * Remember we just get a pointer (reference if you like) here\r
 913      * and so if we store it anywhere, we don't set any pointers to auto free it.\r
 914      */\r
 915     tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);\r
 916 \r
 917     while (tok != NULL && tok->type != ANTLR3_TOKEN_EOF)\r
 918     {\r
 919         discard = ANTLR3_FALSE; /* Assume we are not discarding */\r
 920 \r
 921         /* I employ a bit of a trick, or perhaps hack here. Rather than\r
 922          * store a pointer to a structure in the override map and discard set\r
 923          * we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0\r
 924          * we can distinguish "not being there" from "being channel or type 0"\r
 925          */\r
 926 \r
 927         if (tokenStream->discardSet != NULL\r
 928             && tokenStream->discardSet->get(tokenStream->discardSet, tok->getType(tok)) != NULL)\r
 929         {\r
 930             discard = ANTLR3_TRUE;\r
 931         }\r
 932         else if (   tokenStream->discardOffChannel == ANTLR3_TRUE\r
 933                  && tok->getChannel(tok) != tokenStream->channel\r
 934                  )\r
 935         {\r
 936             discard = ANTLR3_TRUE;\r
 937         }\r
 938         else if (tokenStream->channelOverrides != NULL)\r
 939         {\r
 940             /* See if this type is in the override map\r
 941              */\r
 942             channelI = tokenStream->channelOverrides->get(tokenStream->channelOverrides, tok->getType(tok) + 1);\r
 943 \r
 944             if (channelI != NULL)\r
 945             {\r
 946                 /* Override found\r
 947                  */\r
 948                 tok->setChannel(tok, ANTLR3_UINT32_CAST(channelI) - 1);\r
 949             }\r
 950         }\r
 951 \r
 952         /* If not discarding it, add it to the list at the current index\r
 953          */\r
 954         if (discard == ANTLR3_FALSE)\r
 955         {\r
 956             /* Add it, indicating that we will delete it and the table should not\r
 957              */\r
 958             tok->setTokenIndex(tok, index);\r
 959             tokenStream->p++;\r
 960             tokenStream->tokens->add(tokenStream->tokens, (void *) tok, NULL);\r
 961             index++;\r
 962         }\r
 963 \r
 964         tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);\r
 965     }\r
 966 \r
 967     /* Cache the size so we don't keep doing indirect method calls. We do this as\r
 968      * early as possible so that anything after this may utilize the cached value.\r
 969      */\r
 970     tokenStream->tstream->istream->cachedSize = tokenStream->tokens->count;\r
 971 \r
 972     /* Set the consume pointer to the first token that is on our channel\r
 973      */\r
 974     tokenStream->p = 0;\r
 975     tokenStream->p = skipOffTokenChannels(tokenStream, tokenStream->p);\r
 976 \r
 977 }\r
 978 \r
 979 /// Given a starting index, return the index of the first on-channel\r
 980 ///  token.\r
 981 ///\r
 982 static ANTLR3_UINT32\r
 983 skipOffTokenChannels(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i) {\r
 984     ANTLR3_INT32 n;\r
 985     pANTLR3_COMMON_TOKEN tok;\r
 986 \r
 987     n = tokenStream->tstream->istream->cachedSize;\r
 988 \r
 989     while (i < n)\r
 990     {\r
 991         tok =  (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[i].element;\r
 992 \r
 993         if (tok->channel!= tokenStream->channel)\r
 994         {\r
 995             i++;\r
 996         }\r
 997         else\r
 998         {\r
 999             return i;\r
1000         }\r
1001     }\r
1002     return i;\r
1003 }\r
1004 \r
1005 static ANTLR3_UINT32\r
1006 skipOffTokenChannelsReverse(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 x)\r
1007 {\r
1008     pANTLR3_COMMON_TOKEN tok;\r
1009 \r
1010     while (x >= 0)\r
1011     {\r
1012         tok =  (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[x].element;\r
1013         \r
1014         if ((tok->channel != tokenStream->channel))\r
1015         {\r
1016             x--;\r
1017         }\r
1018         else\r
1019         {\r
1020             return x;\r
1021         }\r
1022     }\r
1023     return x;\r
1024 }\r
1025 \r
1026 /// Return a string that represents the name assoicated with the input source\r
1027 ///\r
1028 /// /param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream.\r
1029 ///\r
1030 /// /returns \r
1031 /// /implements ANTLR3_INT_STREAM_struct::getSourceName()\r
1032 ///\r
1033 static pANTLR3_STRING           \r
1034 getSourceName                           (pANTLR3_INT_STREAM is)\r
1035 {\r
1036         // Slightly convoluted as we must trace back to the lexer's input source\r
1037         // via the token source. The streamName that is here is not initialized\r
1038         // because this is a token stream, not a file or string stream, which are the\r
1039         // only things that have a context for a source name.\r
1040         //\r
1041         return ((pANTLR3_TOKEN_STREAM)(is->super))->tokenSource->fileName;\r
1042 }\r