-/// \file \r
-/// Default implementation of CommonTokenStream\r
-///\r
-\r
-// [The "BSD licence"]\r
-// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC\r
-// http://www.temporal-wave.com\r
-// http://www.linkedin.com/in/jimidle\r
-//\r
-// All rights reserved.\r
-//\r
-// Redistribution and use in source and binary forms, with or without\r
-// modification, are permitted provided that the following conditions\r
-// are met:\r
-// 1. Redistributions of source code must retain the above copyright\r
-// notice, this list of conditions and the following disclaimer.\r
-// 2. Redistributions in binary form must reproduce the above copyright\r
-// notice, this list of conditions and the following disclaimer in the\r
-// documentation and/or other materials provided with the distribution.\r
-// 3. The name of the author may not be used to endorse or promote products\r
-// derived from this software without specific prior written permission.\r
-//\r
-// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\r
-// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\r
-// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\r
-// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\r
-// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\r
-// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\r
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\r
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\r
-// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r
-\r
-#include <antlr3tokenstream.h>\r
-\r
-#ifdef ANTLR3_WINDOWS\r
-#pragma warning( disable : 4100 )\r
-#endif\r
-\r
-// COMMON_TOKEN_STREAM API\r
-//\r
-static void setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel);\r
-static void discardTokenType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 ttype);\r
-static void discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_BOOLEAN discard);\r
-static pANTLR3_VECTOR getTokens (pANTLR3_COMMON_TOKEN_STREAM cts);\r
-static pANTLR3_LIST getTokenRange (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);\r
-static pANTLR3_LIST getTokensSet (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types);\r
-static pANTLR3_LIST getTokensList (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list);\r
-static pANTLR3_LIST getTokensType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type);\r
-\r
-// TOKEN_STREAM API \r
-//\r
-static pANTLR3_COMMON_TOKEN tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);\r
-static pANTLR3_COMMON_TOKEN dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);\r
-static pANTLR3_COMMON_TOKEN get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i);\r
-static pANTLR3_TOKEN_SOURCE getTokenSource (pANTLR3_TOKEN_STREAM ts);\r
-static void setTokenSource (pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource);\r
-static pANTLR3_STRING toString (pANTLR3_TOKEN_STREAM ts);\r
-static pANTLR3_STRING toStringSS (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);\r
-static pANTLR3_STRING toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop);\r
-static void setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger);\r
-\r
-// INT STREAM API\r
-//\r
-static void consume (pANTLR3_INT_STREAM is);\r
-static void dbgConsume (pANTLR3_INT_STREAM is);\r
-static ANTLR3_UINT32 _LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i);\r
-static ANTLR3_UINT32 dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i);\r
-static ANTLR3_MARKER mark (pANTLR3_INT_STREAM is);\r
-static ANTLR3_MARKER dbgMark (pANTLR3_INT_STREAM is);\r
-static void release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark);\r
-static ANTLR3_UINT32 size (pANTLR3_INT_STREAM is);\r
-static ANTLR3_MARKER tindex (pANTLR3_INT_STREAM is);\r
-static void rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);\r
-static void dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);\r
-static void rewindLast (pANTLR3_INT_STREAM is);\r
-static void dbgRewindLast (pANTLR3_INT_STREAM is);\r
-static void seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index);\r
-static void dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index);\r
-static pANTLR3_STRING getSourceName (pANTLR3_INT_STREAM is);\r
-static void antlr3TokenStreamFree (pANTLR3_TOKEN_STREAM stream);\r
-static void antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream);\r
-\r
-// Helpers\r
-//\r
-static void fillBuffer (pANTLR3_COMMON_TOKEN_STREAM tokenStream);\r
-static ANTLR3_UINT32 skipOffTokenChannels (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);\r
-static ANTLR3_UINT32 skipOffTokenChannelsReverse (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);\r
-static pANTLR3_COMMON_TOKEN LB (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);\r
-\r
-ANTLR3_API pANTLR3_TOKEN_STREAM\r
-antlr3TokenStreamNew()\r
-{\r
- pANTLR3_TOKEN_STREAM stream;\r
-\r
- // Memory for the interface structure\r
- //\r
- stream = (pANTLR3_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_STREAM));\r
-\r
- if (stream == NULL)\r
- {\r
- return NULL;\r
- }\r
-\r
- // Install basic API \r
- //\r
- stream->free = antlr3TokenStreamFree;\r
-\r
- \r
- return stream;\r
-}\r
-\r
-static void\r
-antlr3TokenStreamFree(pANTLR3_TOKEN_STREAM stream)\r
-{ \r
- ANTLR3_FREE(stream);\r
-}\r
-\r
-static void \r
-antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream)\r
-{\r
- // We only free up our subordinate interfaces if they belong\r
- // to us, otherwise we let whoever owns them deal with them.\r
- //\r
- if (stream->tstream->super == stream)\r
- {\r
- if (stream->tstream->istream->super == stream->tstream)\r
- {\r
- stream->tstream->istream->free(stream->tstream->istream);\r
- stream->tstream->istream = NULL;\r
- }\r
- stream->tstream->free(stream->tstream);\r
- }\r
-\r
- // Now we free our own resources\r
- //\r
- if (stream->tokens != NULL)\r
- {\r
- stream->tokens->free(stream->tokens);\r
- stream->tokens = NULL;\r
- }\r
- if (stream->discardSet != NULL)\r
- {\r
- stream->discardSet->free(stream->discardSet);\r
- stream->discardSet = NULL;\r
- }\r
- if (stream->channelOverrides != NULL)\r
- {\r
- stream->channelOverrides->free(stream->channelOverrides);\r
- stream->channelOverrides = NULL;\r
- }\r
-\r
- // Free our memory now\r
- //\r
- ANTLR3_FREE(stream);\r
-}\r
-\r
-ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM\r
-antlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger)\r
-{\r
- pANTLR3_COMMON_TOKEN_STREAM stream;\r
-\r
- // Create a standard token stream\r
- //\r
- stream = antlr3CommonTokenStreamSourceNew(hint, source);\r
-\r
- // Install the debugger object\r
- //\r
- stream->tstream->debugger = debugger;\r
-\r
- // Override standard token stream methods with debugging versions\r
- //\r
- stream->tstream->initialStreamState = ANTLR3_FALSE;\r
-\r
- stream->tstream->_LT = dbgTokLT;\r
-\r
- stream->tstream->istream->consume = dbgConsume;\r
- stream->tstream->istream->_LA = dbgLA;\r
- stream->tstream->istream->mark = dbgMark;\r
- stream->tstream->istream->rewind = dbgRewindStream;\r
- stream->tstream->istream->rewindLast = dbgRewindLast;\r
- stream->tstream->istream->seek = dbgSeek;\r
-\r
- return stream;\r
-}\r
-\r
-ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM\r
-antlr3CommonTokenStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source)\r
-{\r
- pANTLR3_COMMON_TOKEN_STREAM stream;\r
-\r
- stream = antlr3CommonTokenStreamNew(hint);\r
-\r
- stream->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;\r
- \r
- stream->channelOverrides = NULL;\r
- stream->discardSet = NULL;\r
- stream->discardOffChannel = ANTLR3_FALSE;\r
-\r
- stream->tstream->setTokenSource(stream->tstream, source);\r
-\r
- stream->free = antlr3CTSFree;\r
- return stream;\r
-}\r
-\r
-ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM\r
-antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint)\r
-{\r
- pANTLR3_COMMON_TOKEN_STREAM stream;\r
-\r
- /* Memory for the interface structure\r
- */\r
- stream = (pANTLR3_COMMON_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TOKEN_STREAM));\r
-\r
- if (stream == NULL)\r
- {\r
- return NULL;\r
- }\r
-\r
- /* Create space for the token stream interface\r
- */\r
- stream->tstream = antlr3TokenStreamNew();\r
- stream->tstream->super = stream;\r
-\r
- /* Create space for the INT_STREAM interfacce\r
- */\r
- stream->tstream->istream = antlr3IntStreamNew();\r
- stream->tstream->istream->super = (stream->tstream);\r
- stream->tstream->istream->type = ANTLR3_TOKENSTREAM;\r
-\r
- /* Install the token tracking tables\r
- */\r
- stream->tokens = antlr3VectorNew(0);\r
-\r
- /* Defaults\r
- */\r
- stream->p = -1;\r
-\r
- /* Install the common token stream API\r
- */\r
- stream->setTokenTypeChannel = setTokenTypeChannel;\r
- stream->discardTokenType = discardTokenType;\r
- stream->discardOffChannelToks = discardOffChannel;\r
- stream->getTokens = getTokens;\r
- stream->getTokenRange = getTokenRange;\r
- stream->getTokensSet = getTokensSet;\r
- stream->getTokensList = getTokensList;\r
- stream->getTokensType = getTokensType;\r
-\r
- /* Install the token stream API\r
- */\r
- stream->tstream->_LT = tokLT;\r
- stream->tstream->get = get;\r
- stream->tstream->getTokenSource = getTokenSource;\r
- stream->tstream->setTokenSource = setTokenSource;\r
- stream->tstream->toString = toString;\r
- stream->tstream->toStringSS = toStringSS;\r
- stream->tstream->toStringTT = toStringTT;\r
- stream->tstream->setDebugListener = setDebugListener;\r
-\r
- /* Install INT_STREAM interface\r
- */\r
- stream->tstream->istream->_LA = _LA;\r
- stream->tstream->istream->mark = mark;\r
- stream->tstream->istream->release = release;\r
- stream->tstream->istream->size = size;\r
- stream->tstream->istream->index = tindex;\r
- stream->tstream->istream->rewind = rewindStream;\r
- stream->tstream->istream->rewindLast= rewindLast;\r
- stream->tstream->istream->seek = seek;\r
- stream->tstream->istream->consume = consume;\r
- stream->tstream->istream->getSourceName = getSourceName;\r
-\r
- return stream;\r
-}\r
-\r
-// Install a debug listener adn switch to debug mode methods\r
-//\r
-static void \r
-setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger)\r
-{\r
- // Install the debugger object\r
- //\r
- ts->debugger = debugger;\r
-\r
- // Override standard token stream methods with debugging versions\r
- //\r
- ts->initialStreamState = ANTLR3_FALSE;\r
-\r
- ts->_LT = dbgTokLT;\r
-\r
- ts->istream->consume = dbgConsume;\r
- ts->istream->_LA = dbgLA;\r
- ts->istream->mark = dbgMark;\r
- ts->istream->rewind = dbgRewindStream;\r
- ts->istream->rewindLast = dbgRewindLast;\r
- ts->istream->seek = dbgSeek;\r
-}\r
-\r
-/** Get the ith token from the current position 1..n where k=1 is the\r
-* first symbol of lookahead.\r
-*/\r
-static pANTLR3_COMMON_TOKEN \r
-tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)\r
-{\r
- ANTLR3_INT32 i;\r
- ANTLR3_INT32 n;\r
- pANTLR3_COMMON_TOKEN_STREAM cts;\r
-\r
- cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;\r
-\r
- if (k < 0)\r
- {\r
- return LB(cts, -k);\r
- }\r
-\r
- if (cts->p == -1)\r
- {\r
- fillBuffer(cts);\r
- }\r
- if (k == 0)\r
- {\r
- return NULL;\r
- }\r
-\r
- if ((cts->p + k - 1) >= (ANTLR3_INT32)ts->istream->cachedSize)\r
- {\r
- pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken);\r
-\r
- teof->setStartIndex (teof, ts->istream->index (ts->istream));\r
- teof->setStopIndex (teof, ts->istream->index (ts->istream));\r
- return teof;\r
- }\r
-\r
- i = cts->p;\r
- n = 1;\r
-\r
- /* Need to find k good tokens, skipping ones that are off channel\r
- */\r
- while ( n < k)\r
- {\r
- /* Skip off-channel tokens */\r
- i = skipOffTokenChannels(cts, i+1); /* leave p on valid token */\r
- n++;\r
- }\r
- if ( (ANTLR3_UINT32) i >= ts->istream->cachedSize)\r
- {\r
- pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken);\r
-\r
- teof->setStartIndex (teof, ts->istream->index(ts->istream));\r
- teof->setStopIndex (teof, ts->istream->index(ts->istream));\r
- return teof;\r
- }\r
-\r
- // Here the token must be in the input vector. Rather then incut\r
- // function call penalty, we jsut return the pointer directly\r
- // from the vector\r
- //\r
- return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;\r
- //return (pANTLR3_COMMON_TOKEN)cts->tokens->get(cts->tokens, i);\r
-}\r
-\r
-/// Debug only method to flag consumption of initial off-channel\r
-/// tokens in the input stream\r
-///\r
-static void\r
-consumeInitialHiddenTokens(pANTLR3_INT_STREAM is)\r
-{\r
- ANTLR3_MARKER first;\r
- ANTLR3_INT32 i;\r
- pANTLR3_TOKEN_STREAM ts;\r
-\r
- ts = (pANTLR3_TOKEN_STREAM) is->super;\r
- first = is->index(is);\r
-\r
- for (i=0; i<first; i++)\r
- {\r
- ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, i));\r
- }\r
-\r
- ts->initialStreamState = ANTLR3_FALSE;\r
-\r
-}\r
-\r
-/// As per the normal tokLT but sends information to the debugger\r
-///\r
-static pANTLR3_COMMON_TOKEN \r
-dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)\r
-{\r
- if (ts->initialStreamState == ANTLR3_TRUE)\r
- {\r
- consumeInitialHiddenTokens(ts->istream);\r
- }\r
- return tokLT(ts, k);\r
-}\r
-\r
-#ifdef ANTLR3_WINDOWS\r
- /* When fully optimized VC7 complains about non reachable code.\r
- * Not yet sure if this is an optimizer bug, or a bug in the flow analysis\r
- */\r
-#pragma warning( disable : 4702 )\r
-#endif\r
-\r
-static pANTLR3_COMMON_TOKEN\r
-LB(pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 k)\r
-{\r
- ANTLR3_INT32 i;\r
- ANTLR3_INT32 n;\r
-\r
- if (cts->p == -1)\r
- {\r
- fillBuffer(cts);\r
- }\r
- if (k == 0)\r
- {\r
- return NULL;\r
- }\r
- if ((cts->p - k) < 0)\r
- {\r
- return NULL;\r
- }\r
-\r
- i = cts->p;\r
- n = 1;\r
-\r
- /* Need to find k good tokens, going backwards, skipping ones that are off channel\r
- */\r
- while (n <= (ANTLR3_INT32) k)\r
- {\r
- /* Skip off-channel tokens\r
- */\r
-\r
- i = skipOffTokenChannelsReverse(cts, i - 1); /* leave p on valid token */\r
- n++;\r
- }\r
- if (i < 0)\r
- {\r
- return NULL;\r
- }\r
- // Here the token must be in the input vector. Rather then incut\r
- // function call penalty, we jsut return the pointer directly\r
- // from the vector\r
- //\r
- return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;\r
-}\r
-\r
-static pANTLR3_COMMON_TOKEN \r
-get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i)\r
-{\r
- pANTLR3_COMMON_TOKEN_STREAM cts;\r
-\r
- cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;\r
-\r
- return (pANTLR3_COMMON_TOKEN)(cts->tokens->get(cts->tokens, i)); /* Token index is zero based but vectors are 1 based */\r
-}\r
-\r
-static pANTLR3_TOKEN_SOURCE \r
-getTokenSource (pANTLR3_TOKEN_STREAM ts)\r
-{\r
- return ts->tokenSource;\r
-}\r
-\r
-static void\r
-setTokenSource ( pANTLR3_TOKEN_STREAM ts,\r
- pANTLR3_TOKEN_SOURCE tokenSource)\r
-{\r
- ts->tokenSource = tokenSource;\r
-}\r
-\r
-static pANTLR3_STRING \r
-toString (pANTLR3_TOKEN_STREAM ts)\r
-{\r
- pANTLR3_COMMON_TOKEN_STREAM cts;\r
-\r
- cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;\r
-\r
- if (cts->p == -1)\r
- {\r
- fillBuffer(cts);\r
- }\r
-\r
- return ts->toStringSS(ts, 0, ts->istream->size(ts->istream));\r
-}\r
-\r
-static pANTLR3_STRING\r
-toStringSS(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)\r
-{\r
- pANTLR3_STRING string;\r
- pANTLR3_TOKEN_SOURCE tsource;\r
- pANTLR3_COMMON_TOKEN tok;\r
- ANTLR3_UINT32 i;\r
- pANTLR3_COMMON_TOKEN_STREAM cts;\r
-\r
- cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;\r
-\r
- if (cts->p == -1)\r
- {\r
- fillBuffer(cts);\r
- }\r
- if (stop >= ts->istream->size(ts->istream))\r
- {\r
- stop = ts->istream->size(ts->istream) - 1;\r
- }\r
-\r
- /* Who is giving us these tokens?\r
- */\r
- tsource = ts->getTokenSource(ts);\r
-\r
- if (tsource != NULL && cts->tokens != NULL)\r
- {\r
- /* Finally, let's get a string\r
- */\r
- string = tsource->strFactory->newRaw(tsource->strFactory);\r
-\r
- for (i = start; i <= stop; i++)\r
- {\r
- tok = ts->get(ts, i);\r
- if (tok != NULL)\r
- {\r
- string->appendS(string, tok->getText(tok));\r
- }\r
- }\r
-\r
- return string;\r
- }\r
- return NULL;\r
-\r
-}\r
-\r
-static pANTLR3_STRING \r
-toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop)\r
-{\r
- if (start != NULL && stop != NULL)\r
- {\r
- return ts->toStringSS(ts, (ANTLR3_UINT32)start->getTokenIndex(start), (ANTLR3_UINT32)stop->getTokenIndex(stop));\r
- }\r
- else\r
- {\r
- return NULL;\r
- }\r
-}\r
-\r
-/** Move the input pointer to the next incoming token. The stream\r
- * must become active with LT(1) available. consume() simply\r
- * moves the input pointer so that LT(1) points at the next\r
- * input symbol. Consume at least one token.\r
- *\r
- * Walk past any token not on the channel the parser is listening to.\r
- */\r
-static void \r
-consume (pANTLR3_INT_STREAM is)\r
-{\r
- pANTLR3_COMMON_TOKEN_STREAM cts;\r
- pANTLR3_TOKEN_STREAM ts;\r
-\r
- ts = (pANTLR3_TOKEN_STREAM) is->super;\r
- cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;\r
-\r
- if ((ANTLR3_UINT32)cts->p < cts->tokens->size(cts->tokens))\r
- {\r
- cts->p++;\r
- cts->p = skipOffTokenChannels(cts, cts->p);\r
- }\r
-}\r
-\r
-\r
-/// As per ordinary consume but notifies the debugger about hidden\r
-/// tokens and so on.\r
-///\r
-static void\r
-dbgConsume (pANTLR3_INT_STREAM is)\r
-{\r
- pANTLR3_TOKEN_STREAM ts;\r
- ANTLR3_MARKER a;\r
- ANTLR3_MARKER b;\r
- pANTLR3_COMMON_TOKEN t;\r
-\r
- ts = (pANTLR3_TOKEN_STREAM) is->super;\r
-\r
- if (ts->initialStreamState == ANTLR3_TRUE)\r
- {\r
- consumeInitialHiddenTokens(is);\r
- }\r
- \r
- a = is->index(is); // Where are we right now?\r
- t = ts->_LT(ts, 1); // Current token from stream\r
-\r
- consume(is); // Standard consumer\r
-\r
- b = is->index(is); // Where are we after consuming 1 on channel token?\r
-\r
- ts->debugger->consumeToken(ts->debugger, t); // Tell the debugger that we consumed the first token\r
-\r
- if (b>a+1)\r
- {\r
- // The standard consume caused the index to advance by more than 1,\r
- // which can only happen if it skipped some off-channel tokens.\r
- // we need to tell the debugger about those tokens.\r
- //\r
- ANTLR3_MARKER i;\r
-\r
- for (i = a+1; i<b; i++)\r
- {\r
- ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, (ANTLR3_UINT32)i));\r
- }\r
-\r
- }\r
-}\r
-\r
-/** A simple filter mechanism whereby you can tell this token stream\r
- * to force all tokens of type ttype to be on channel. For example,\r
- * when interpreting, we cannot execute actions so we need to tell\r
- * the stream to force all WS and NEWLINE to be a different, ignored,\r
- * channel.\r
- */\r
-static void \r
-setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel)\r
-{\r
- if (tokenStream->channelOverrides == NULL)\r
- {\r
- tokenStream->channelOverrides = antlr3ListNew(10);\r
- }\r
-\r
- /* We add one to the channel so we can distinguish NULL as being no entry in the\r
- * table for a particular token type.\r
- */\r
- tokenStream->channelOverrides->put(tokenStream->channelOverrides, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)channel + 1), NULL);\r
-}\r
-\r
-static void \r
-discardTokenType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 ttype)\r
-{\r
- if (tokenStream->discardSet == NULL)\r
- {\r
- tokenStream->discardSet = antlr3ListNew(31);\r
- }\r
-\r
- /* We add one to the channel so we can distinguish NULL as being no entry in the\r
- * table for a particular token type. We could use bitsets for this I suppose too.\r
- */\r
- tokenStream->discardSet->put(tokenStream->discardSet, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)ttype + 1), NULL);\r
-}\r
-\r
-static void \r
-discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_BOOLEAN discard)\r
-{\r
- tokenStream->discardOffChannel = discard;\r
-}\r
-\r
-static pANTLR3_VECTOR \r
-getTokens (pANTLR3_COMMON_TOKEN_STREAM tokenStream)\r
-{\r
- if (tokenStream->p == -1)\r
- {\r
- fillBuffer(tokenStream);\r
- }\r
-\r
- return tokenStream->tokens;\r
-}\r
-\r
-static pANTLR3_LIST \r
-getTokenRange (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)\r
-{\r
- return tokenStream->getTokensSet(tokenStream, start, stop, NULL);\r
-} \r
-/** Given a start and stop index, return a List of all tokens in\r
- * the token type BitSet. Return null if no tokens were found. This\r
- * method looks at both on and off channel tokens.\r
- */\r
-static pANTLR3_LIST \r
-getTokensSet (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types)\r
-{\r
- pANTLR3_LIST filteredList;\r
- ANTLR3_UINT32 i;\r
- ANTLR3_UINT32 n;\r
- pANTLR3_COMMON_TOKEN tok;\r
-\r
- if (tokenStream->p == -1)\r
- {\r
- fillBuffer(tokenStream);\r
- }\r
- if (stop > tokenStream->tstream->istream->size(tokenStream->tstream->istream))\r
- {\r
- stop = tokenStream->tstream->istream->size(tokenStream->tstream->istream);\r
- }\r
- if (start > stop)\r
- {\r
- return NULL;\r
- }\r
-\r
- /* We have the range set, now we need to iterate through the\r
- * installed tokens and create a new list with just the ones we want\r
- * in it. We are just moving pointers about really.\r
- */\r
- filteredList = antlr3ListNew((ANTLR3_UINT32)tokenStream->tstream->istream->size(tokenStream->tstream->istream));\r
-\r
- for (i = start, n = 0; i<= stop; i++)\r
- {\r
- tok = tokenStream->tstream->get(tokenStream->tstream, i);\r
-\r
- if ( types == NULL\r
- || types->isMember(types, tok->getType(tok) == ANTLR3_TRUE)\r
- )\r
- {\r
- filteredList->put(filteredList, n++, (void *)tok, NULL);\r
- }\r
- }\r
- \r
- /* Did we get any then?\r
- */\r
- if (filteredList->size(filteredList) == 0)\r
- {\r
- filteredList->free(filteredList);\r
- filteredList = NULL;\r
- }\r
-\r
- return filteredList;\r
-}\r
-\r
-static pANTLR3_LIST \r
-getTokensList (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list)\r
-{\r
- pANTLR3_BITSET bitSet;\r
- pANTLR3_LIST newlist;\r
-\r
- bitSet = antlr3BitsetList(list->table);\r
-\r
- newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);\r
-\r
- bitSet->free(bitSet);\r
-\r
- return newlist;\r
-\r
-}\r
-\r
-static pANTLR3_LIST \r
-getTokensType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type)\r
-{\r
- pANTLR3_BITSET bitSet;\r
- pANTLR3_LIST newlist;\r
-\r
- bitSet = antlr3BitsetOf(type, -1);\r
- newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);\r
-\r
- bitSet->free(bitSet);\r
-\r
- return newlist;\r
-}\r
-\r
-static ANTLR3_UINT32 \r
-_LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)\r
-{\r
- pANTLR3_TOKEN_STREAM ts;\r
- pANTLR3_COMMON_TOKEN tok;\r
-\r
- ts = (pANTLR3_TOKEN_STREAM) is->super;\r
-\r
- tok = ts->_LT(ts, i);\r
-\r
- if (tok != NULL)\r
- {\r
- return tok->getType(tok);\r
- }\r
- else\r
- {\r
- return ANTLR3_TOKEN_INVALID;\r
- }\r
-}\r
-\r
-/// As per _LA() but for debug mode.\r
-///\r
-static ANTLR3_UINT32 \r
-dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)\r
-{\r
- pANTLR3_TOKEN_STREAM ts;\r
- \r
- ts = (pANTLR3_TOKEN_STREAM) is->super;\r
-\r
- if (ts->initialStreamState == ANTLR3_TRUE)\r
- {\r
- consumeInitialHiddenTokens(is);\r
- }\r
- ts->debugger->LT(ts->debugger, i, tokLT(ts, i));\r
- return _LA(is, i);\r
-}\r
-\r
-static ANTLR3_MARKER\r
-mark (pANTLR3_INT_STREAM is)\r
-{\r
- is->lastMarker = is->index(is);\r
- return is->lastMarker;\r
-}\r
-\r
-/// As per mark() but with a call to tell the debugger we are doing this\r
-///\r
-static ANTLR3_MARKER\r
-dbgMark (pANTLR3_INT_STREAM is)\r
-{\r
- pANTLR3_TOKEN_STREAM ts;\r
- \r
- ts = (pANTLR3_TOKEN_STREAM) is->super;\r
- \r
- is->lastMarker = is->index(is);\r
- ts->debugger->mark(ts->debugger, is->lastMarker);\r
-\r
- return is->lastMarker;\r
-}\r
-\r
-static void \r
-release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark)\r
-{\r
- return;\r
-}\r
-\r
-static ANTLR3_UINT32 \r
-size (pANTLR3_INT_STREAM is)\r
-{\r
- pANTLR3_COMMON_TOKEN_STREAM cts;\r
- pANTLR3_TOKEN_STREAM ts;\r
-\r
- if (is->cachedSize > 0)\r
- {\r
- return is->cachedSize;\r
- }\r
- ts = (pANTLR3_TOKEN_STREAM) is->super;\r
- cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;\r
-\r
- is->cachedSize = cts->tokens->count;\r
- return is->cachedSize;\r
-}\r
-\r
-static ANTLR3_MARKER \r
-tindex (pANTLR3_INT_STREAM is)\r
-{\r
- pANTLR3_COMMON_TOKEN_STREAM cts;\r
- pANTLR3_TOKEN_STREAM ts;\r
-\r
- ts = (pANTLR3_TOKEN_STREAM) is->super;\r
- cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;\r
-\r
- return cts->p;\r
-}\r
-\r
-static void \r
-dbgRewindLast (pANTLR3_INT_STREAM is)\r
-{\r
- pANTLR3_TOKEN_STREAM ts;\r
-\r
- ts = (pANTLR3_TOKEN_STREAM) is->super;\r
-\r
- ts->debugger->rewindLast(ts->debugger);\r
-\r
- is->rewind(is, is->lastMarker);\r
-}\r
-static void \r
-rewindLast (pANTLR3_INT_STREAM is)\r
-{\r
- is->rewind(is, is->lastMarker);\r
-}\r
-static void \r
-rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)\r
-{\r
- is->seek(is, (ANTLR3_UINT32)(marker));\r
-}\r
-static void \r
-dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)\r
-{\r
- pANTLR3_TOKEN_STREAM ts;\r
-\r
- ts = (pANTLR3_TOKEN_STREAM) is->super;\r
-\r
- ts->debugger->rewind(ts->debugger, marker);\r
-\r
- is->seek(is, (ANTLR3_UINT32)(marker));\r
-}\r
-\r
-static void \r
-seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)\r
-{\r
- pANTLR3_COMMON_TOKEN_STREAM cts;\r
- pANTLR3_TOKEN_STREAM ts;\r
-\r
- ts = (pANTLR3_TOKEN_STREAM) is->super;\r
- cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;\r
-\r
- cts->p = (ANTLR3_UINT32)index;\r
-}\r
-static void \r
-dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)\r
-{\r
- // TODO: Implement seek in debugger when Ter adds it to Java\r
- //\r
- seek(is, index);\r
-}\r
-ANTLR3_API void\r
-fillBufferExt(pANTLR3_COMMON_TOKEN_STREAM tokenStream)\r
-{\r
- fillBuffer(tokenStream);\r
-}\r
-static void\r
-fillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream) {\r
- ANTLR3_UINT32 index;\r
- pANTLR3_COMMON_TOKEN tok;\r
- ANTLR3_BOOLEAN discard;\r
- void * channelI;\r
-\r
- /* Start at index 0 of course\r
- */\r
- index = 0;\r
-\r
- /* Pick out the next token from the token source\r
- * Remember we just get a pointer (reference if you like) here\r
- * and so if we store it anywhere, we don't set any pointers to auto free it.\r
- */\r
- tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);\r
-\r
- while (tok != NULL && tok->type != ANTLR3_TOKEN_EOF)\r
- {\r
- discard = ANTLR3_FALSE; /* Assume we are not discarding */\r
-\r
- /* I employ a bit of a trick, or perhaps hack here. Rather than\r
- * store a pointer to a structure in the override map and discard set\r
- * we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0\r
- * we can distinguish "not being there" from "being channel or type 0"\r
- */\r
-\r
- if (tokenStream->discardSet != NULL\r
- && tokenStream->discardSet->get(tokenStream->discardSet, tok->getType(tok)) != NULL)\r
- {\r
- discard = ANTLR3_TRUE;\r
- }\r
- else if ( tokenStream->discardOffChannel == ANTLR3_TRUE\r
- && tok->getChannel(tok) != tokenStream->channel\r
- )\r
- {\r
- discard = ANTLR3_TRUE;\r
- }\r
- else if (tokenStream->channelOverrides != NULL)\r
- {\r
- /* See if this type is in the override map\r
- */\r
- channelI = tokenStream->channelOverrides->get(tokenStream->channelOverrides, tok->getType(tok) + 1);\r
-\r
- if (channelI != NULL)\r
- {\r
- /* Override found\r
- */\r
- tok->setChannel(tok, ANTLR3_UINT32_CAST(channelI) - 1);\r
- }\r
- }\r
-\r
- /* If not discarding it, add it to the list at the current index\r
- */\r
- if (discard == ANTLR3_FALSE)\r
- {\r
- /* Add it, indicating that we will delete it and the table should not\r
- */\r
- tok->setTokenIndex(tok, index);\r
- tokenStream->p++;\r
- tokenStream->tokens->add(tokenStream->tokens, (void *) tok, NULL);\r
- index++;\r
- }\r
-\r
- tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);\r
- }\r
-\r
- /* Cache the size so we don't keep doing indirect method calls. We do this as\r
- * early as possible so that anything after this may utilize the cached value.\r
- */\r
- tokenStream->tstream->istream->cachedSize = tokenStream->tokens->count;\r
-\r
- /* Set the consume pointer to the first token that is on our channel\r
- */\r
- tokenStream->p = 0;\r
- tokenStream->p = skipOffTokenChannels(tokenStream, tokenStream->p);\r
-\r
-}\r
-\r
-/// Given a starting index, return the index of the first on-channel\r
-/// token.\r
-///\r
-static ANTLR3_UINT32\r
-skipOffTokenChannels(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i) {\r
- ANTLR3_INT32 n;\r
- pANTLR3_COMMON_TOKEN tok;\r
-\r
- n = tokenStream->tstream->istream->cachedSize;\r
-\r
- while (i < n)\r
- {\r
- tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[i].element;\r
-\r
- if (tok->channel!= tokenStream->channel)\r
- {\r
- i++;\r
- }\r
- else\r
- {\r
- return i;\r
- }\r
- }\r
- return i;\r
-}\r
-\r
-static ANTLR3_UINT32\r
-skipOffTokenChannelsReverse(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 x)\r
-{\r
- pANTLR3_COMMON_TOKEN tok;\r
-\r
- while (x >= 0)\r
- {\r
- tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[x].element;\r
- \r
- if ((tok->channel != tokenStream->channel))\r
- {\r
- x--;\r
- }\r
- else\r
- {\r
- return x;\r
- }\r
- }\r
- return x;\r
-}\r
-\r
-/// Return a string that represents the name assoicated with the input source\r
-///\r
-/// /param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream.\r
-///\r
-/// /returns \r
-/// /implements ANTLR3_INT_STREAM_struct::getSourceName()\r
-///\r
-static pANTLR3_STRING \r
-getSourceName (pANTLR3_INT_STREAM is)\r
-{\r
- // Slightly convoluted as we must trace back to the lexer's input source\r
- // via the token source. The streamName that is here is not initialized\r
- // because this is a token stream, not a file or string stream, which are the\r
- // only things that have a context for a source name.\r
- //\r
- return ((pANTLR3_TOKEN_STREAM)(is->super))->tokenSource->fileName;\r
-}\r
+/// \file
+/// Default implementation of CommonTokenStream
+///
+
+// [The "BSD licence"]
+// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
+// http://www.temporal-wave.com
+// http://www.linkedin.com/in/jimidle
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. The name of the author may not be used to endorse or promote products
+// derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <antlr3tokenstream.h>
+
+#ifdef ANTLR3_WINDOWS
+#pragma warning( disable : 4100 )
+#endif
+
+// COMMON_TOKEN_STREAM API
+//
+static void setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel);
+static void discardTokenType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 ttype);
+static void discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_BOOLEAN discard);
+static pANTLR3_VECTOR getTokens (pANTLR3_COMMON_TOKEN_STREAM cts);
+static pANTLR3_LIST getTokenRange (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
+static pANTLR3_LIST getTokensSet (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types);
+static pANTLR3_LIST getTokensList (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list);
+static pANTLR3_LIST getTokensType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type);
+
+// TOKEN_STREAM API
+//
+static pANTLR3_COMMON_TOKEN tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);
+static pANTLR3_COMMON_TOKEN dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);
+static pANTLR3_COMMON_TOKEN get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i);
+static pANTLR3_TOKEN_SOURCE getTokenSource (pANTLR3_TOKEN_STREAM ts);
+static void setTokenSource (pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource);
+static pANTLR3_STRING toString (pANTLR3_TOKEN_STREAM ts);
+static pANTLR3_STRING toStringSS (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
+static pANTLR3_STRING toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop);
+static void setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger);
+
+// INT STREAM API
+//
+static void consume (pANTLR3_INT_STREAM is);
+static void dbgConsume (pANTLR3_INT_STREAM is);
+static ANTLR3_UINT32 _LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i);
+static ANTLR3_UINT32 dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i);
+static ANTLR3_MARKER mark (pANTLR3_INT_STREAM is);
+static ANTLR3_MARKER dbgMark (pANTLR3_INT_STREAM is);
+static void release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark);
+static ANTLR3_UINT32 size (pANTLR3_INT_STREAM is);
+static ANTLR3_MARKER tindex (pANTLR3_INT_STREAM is);
+static void rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);
+static void dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);
+static void rewindLast (pANTLR3_INT_STREAM is);
+static void dbgRewindLast (pANTLR3_INT_STREAM is);
+static void seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index);
+static void dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index);
+static pANTLR3_STRING getSourceName (pANTLR3_INT_STREAM is);
+static void antlr3TokenStreamFree (pANTLR3_TOKEN_STREAM stream);
+static void antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream);
+
+// Helpers
+//
+static void fillBuffer (pANTLR3_COMMON_TOKEN_STREAM tokenStream);
+static ANTLR3_UINT32 skipOffTokenChannels (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
+static ANTLR3_UINT32 skipOffTokenChannelsReverse (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
+static pANTLR3_COMMON_TOKEN LB (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
+
+ANTLR3_API pANTLR3_TOKEN_STREAM
+antlr3TokenStreamNew()
+{
+ pANTLR3_TOKEN_STREAM stream;
+
+ // Memory for the interface structure
+ //
+ stream = (pANTLR3_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_STREAM));
+
+ if (stream == NULL)
+ {
+ return NULL;
+ }
+
+ // Install basic API
+ //
+ stream->free = antlr3TokenStreamFree;
+
+
+ return stream;
+}
+
+static void
+antlr3TokenStreamFree(pANTLR3_TOKEN_STREAM stream)
+{
+ ANTLR3_FREE(stream);
+}
+
+static void
+antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream)
+{
+ // We only free up our subordinate interfaces if they belong
+ // to us, otherwise we let whoever owns them deal with them.
+ //
+ if (stream->tstream->super == stream)
+ {
+ if (stream->tstream->istream->super == stream->tstream)
+ {
+ stream->tstream->istream->free(stream->tstream->istream);
+ stream->tstream->istream = NULL;
+ }
+ stream->tstream->free(stream->tstream);
+ }
+
+ // Now we free our own resources
+ //
+ if (stream->tokens != NULL)
+ {
+ stream->tokens->free(stream->tokens);
+ stream->tokens = NULL;
+ }
+ if (stream->discardSet != NULL)
+ {
+ stream->discardSet->free(stream->discardSet);
+ stream->discardSet = NULL;
+ }
+ if (stream->channelOverrides != NULL)
+ {
+ stream->channelOverrides->free(stream->channelOverrides);
+ stream->channelOverrides = NULL;
+ }
+
+ // Free our memory now
+ //
+ ANTLR3_FREE(stream);
+}
+
+ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
+antlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger)
+{
+ pANTLR3_COMMON_TOKEN_STREAM stream;
+
+ // Create a standard token stream
+ //
+ stream = antlr3CommonTokenStreamSourceNew(hint, source);
+
+ // Install the debugger object
+ //
+ stream->tstream->debugger = debugger;
+
+ // Override standard token stream methods with debugging versions
+ //
+ stream->tstream->initialStreamState = ANTLR3_FALSE;
+
+ stream->tstream->_LT = dbgTokLT;
+
+ stream->tstream->istream->consume = dbgConsume;
+ stream->tstream->istream->_LA = dbgLA;
+ stream->tstream->istream->mark = dbgMark;
+ stream->tstream->istream->rewind = dbgRewindStream;
+ stream->tstream->istream->rewindLast = dbgRewindLast;
+ stream->tstream->istream->seek = dbgSeek;
+
+ return stream;
+}
+
+ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
+antlr3CommonTokenStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source)
+{
+ pANTLR3_COMMON_TOKEN_STREAM stream;
+
+ stream = antlr3CommonTokenStreamNew(hint);
+
+ stream->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;
+
+ stream->channelOverrides = NULL;
+ stream->discardSet = NULL;
+ stream->discardOffChannel = ANTLR3_FALSE;
+
+ stream->tstream->setTokenSource(stream->tstream, source);
+
+ stream->free = antlr3CTSFree;
+ return stream;
+}
+
+ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
+antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint)
+{
+ pANTLR3_COMMON_TOKEN_STREAM stream;
+
+ /* Memory for the interface structure
+ */
+ stream = (pANTLR3_COMMON_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TOKEN_STREAM));
+
+ if (stream == NULL)
+ {
+ return NULL;
+ }
+
+ /* Create space for the token stream interface
+ */
+ stream->tstream = antlr3TokenStreamNew();
+ stream->tstream->super = stream;
+
+ /* Create space for the INT_STREAM interfacce
+ */
+ stream->tstream->istream = antlr3IntStreamNew();
+ stream->tstream->istream->super = (stream->tstream);
+ stream->tstream->istream->type = ANTLR3_TOKENSTREAM;
+
+ /* Install the token tracking tables
+ */
+ stream->tokens = antlr3VectorNew(0);
+
+ /* Defaults
+ */
+ stream->p = -1;
+
+ /* Install the common token stream API
+ */
+ stream->setTokenTypeChannel = setTokenTypeChannel;
+ stream->discardTokenType = discardTokenType;
+ stream->discardOffChannelToks = discardOffChannel;
+ stream->getTokens = getTokens;
+ stream->getTokenRange = getTokenRange;
+ stream->getTokensSet = getTokensSet;
+ stream->getTokensList = getTokensList;
+ stream->getTokensType = getTokensType;
+
+ /* Install the token stream API
+ */
+ stream->tstream->_LT = tokLT;
+ stream->tstream->get = get;
+ stream->tstream->getTokenSource = getTokenSource;
+ stream->tstream->setTokenSource = setTokenSource;
+ stream->tstream->toString = toString;
+ stream->tstream->toStringSS = toStringSS;
+ stream->tstream->toStringTT = toStringTT;
+ stream->tstream->setDebugListener = setDebugListener;
+
+ /* Install INT_STREAM interface
+ */
+ stream->tstream->istream->_LA = _LA;
+ stream->tstream->istream->mark = mark;
+ stream->tstream->istream->release = release;
+ stream->tstream->istream->size = size;
+ stream->tstream->istream->index = tindex;
+ stream->tstream->istream->rewind = rewindStream;
+ stream->tstream->istream->rewindLast= rewindLast;
+ stream->tstream->istream->seek = seek;
+ stream->tstream->istream->consume = consume;
+ stream->tstream->istream->getSourceName = getSourceName;
+
+ return stream;
+}
+
+// Install a debug listener adn switch to debug mode methods
+//
+static void
+setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger)
+{
+ // Install the debugger object
+ //
+ ts->debugger = debugger;
+
+ // Override standard token stream methods with debugging versions
+ //
+ ts->initialStreamState = ANTLR3_FALSE;
+
+ ts->_LT = dbgTokLT;
+
+ ts->istream->consume = dbgConsume;
+ ts->istream->_LA = dbgLA;
+ ts->istream->mark = dbgMark;
+ ts->istream->rewind = dbgRewindStream;
+ ts->istream->rewindLast = dbgRewindLast;
+ ts->istream->seek = dbgSeek;
+}
+
+/** Get the ith token from the current position 1..n where k=1 is the
+* first symbol of lookahead.
+*/
+static pANTLR3_COMMON_TOKEN
+tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
+{
+ ANTLR3_INT32 i;
+ ANTLR3_INT32 n;
+ pANTLR3_COMMON_TOKEN_STREAM cts;
+
+ cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
+
+ if (k < 0)
+ {
+ return LB(cts, -k);
+ }
+
+ if (cts->p == -1)
+ {
+ fillBuffer(cts);
+ }
+ if (k == 0)
+ {
+ return NULL;
+ }
+
+ if ((cts->p + k - 1) >= (ANTLR3_INT32)ts->istream->cachedSize)
+ {
+ pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken);
+
+ teof->setStartIndex (teof, ts->istream->index (ts->istream));
+ teof->setStopIndex (teof, ts->istream->index (ts->istream));
+ return teof;
+ }
+
+ i = cts->p;
+ n = 1;
+
+ /* Need to find k good tokens, skipping ones that are off channel
+ */
+ while ( n < k)
+ {
+ /* Skip off-channel tokens */
+ i = skipOffTokenChannels(cts, i+1); /* leave p on valid token */
+ n++;
+ }
+ if ( (ANTLR3_UINT32) i >= ts->istream->cachedSize)
+ {
+ pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken);
+
+ teof->setStartIndex (teof, ts->istream->index(ts->istream));
+ teof->setStopIndex (teof, ts->istream->index(ts->istream));
+ return teof;
+ }
+
+ // Here the token must be in the input vector. Rather then incut
+ // function call penalty, we jsut return the pointer directly
+ // from the vector
+ //
+ return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;
+ //return (pANTLR3_COMMON_TOKEN)cts->tokens->get(cts->tokens, i);
+}
+
+/// Debug only method to flag consumption of initial off-channel
+/// tokens in the input stream
+///
+static void
+consumeInitialHiddenTokens(pANTLR3_INT_STREAM is)
+{
+ ANTLR3_MARKER first;
+ ANTLR3_INT32 i;
+ pANTLR3_TOKEN_STREAM ts;
+
+ ts = (pANTLR3_TOKEN_STREAM) is->super;
+ first = is->index(is);
+
+ for (i=0; i<first; i++)
+ {
+ ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, i));
+ }
+
+ ts->initialStreamState = ANTLR3_FALSE;
+
+}
+
+/// As per the normal tokLT but sends information to the debugger
+///
+static pANTLR3_COMMON_TOKEN
+dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
+{
+ if (ts->initialStreamState == ANTLR3_TRUE)
+ {
+ consumeInitialHiddenTokens(ts->istream);
+ }
+ return tokLT(ts, k);
+}
+
+#ifdef ANTLR3_WINDOWS
+ /* When fully optimized VC7 complains about non reachable code.
+ * Not yet sure if this is an optimizer bug, or a bug in the flow analysis
+ */
+#pragma warning( disable : 4702 )
+#endif
+
+static pANTLR3_COMMON_TOKEN
+LB(pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 k)
+{
+ ANTLR3_INT32 i;
+ ANTLR3_INT32 n;
+
+ if (cts->p == -1)
+ {
+ fillBuffer(cts);
+ }
+ if (k == 0)
+ {
+ return NULL;
+ }
+ if ((cts->p - k) < 0)
+ {
+ return NULL;
+ }
+
+ i = cts->p;
+ n = 1;
+
+ /* Need to find k good tokens, going backwards, skipping ones that are off channel
+ */
+ while (n <= (ANTLR3_INT32) k)
+ {
+ /* Skip off-channel tokens
+ */
+
+ i = skipOffTokenChannelsReverse(cts, i - 1); /* leave p on valid token */
+ n++;
+ }
+ if (i < 0)
+ {
+ return NULL;
+ }
+ // Here the token must be in the input vector. Rather then incut
+ // function call penalty, we jsut return the pointer directly
+ // from the vector
+ //
+ return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;
+}
+
+static pANTLR3_COMMON_TOKEN
+get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i)
+{
+ pANTLR3_COMMON_TOKEN_STREAM cts;
+
+ cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
+
+ return (pANTLR3_COMMON_TOKEN)(cts->tokens->get(cts->tokens, i)); /* Token index is zero based but vectors are 1 based */
+}
+
+static pANTLR3_TOKEN_SOURCE
+getTokenSource (pANTLR3_TOKEN_STREAM ts)
+{
+ return ts->tokenSource;
+}
+
+static void
+setTokenSource ( pANTLR3_TOKEN_STREAM ts,
+ pANTLR3_TOKEN_SOURCE tokenSource)
+{
+ ts->tokenSource = tokenSource;
+}
+
+static pANTLR3_STRING
+toString (pANTLR3_TOKEN_STREAM ts)
+{
+ pANTLR3_COMMON_TOKEN_STREAM cts;
+
+ cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
+
+ if (cts->p == -1)
+ {
+ fillBuffer(cts);
+ }
+
+ return ts->toStringSS(ts, 0, ts->istream->size(ts->istream));
+}
+
+static pANTLR3_STRING
+toStringSS(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
+{
+ pANTLR3_STRING string;
+ pANTLR3_TOKEN_SOURCE tsource;
+ pANTLR3_COMMON_TOKEN tok;
+ ANTLR3_UINT32 i;
+ pANTLR3_COMMON_TOKEN_STREAM cts;
+
+ cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
+
+ if (cts->p == -1)
+ {
+ fillBuffer(cts);
+ }
+ if (stop >= ts->istream->size(ts->istream))
+ {
+ stop = ts->istream->size(ts->istream) - 1;
+ }
+
+ /* Who is giving us these tokens?
+ */
+ tsource = ts->getTokenSource(ts);
+
+ if (tsource != NULL && cts->tokens != NULL)
+ {
+ /* Finally, let's get a string
+ */
+ string = tsource->strFactory->newRaw(tsource->strFactory);
+
+ for (i = start; i <= stop; i++)
+ {
+ tok = ts->get(ts, i);
+ if (tok != NULL)
+ {
+ string->appendS(string, tok->getText(tok));
+ }
+ }
+
+ return string;
+ }
+ return NULL;
+
+}
+
+static pANTLR3_STRING
+toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop)
+{
+ if (start != NULL && stop != NULL)
+ {
+ return ts->toStringSS(ts, (ANTLR3_UINT32)start->getTokenIndex(start), (ANTLR3_UINT32)stop->getTokenIndex(stop));
+ }
+ else
+ {
+ return NULL;
+ }
+}
+
+/** Move the input pointer to the next incoming token. The stream
+ * must become active with LT(1) available. consume() simply
+ * moves the input pointer so that LT(1) points at the next
+ * input symbol. Consume at least one token.
+ *
+ * Walk past any token not on the channel the parser is listening to.
+ */
+static void
+consume (pANTLR3_INT_STREAM is)
+{
+ pANTLR3_COMMON_TOKEN_STREAM cts;
+ pANTLR3_TOKEN_STREAM ts;
+
+ ts = (pANTLR3_TOKEN_STREAM) is->super;
+ cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
+
+ if ((ANTLR3_UINT32)cts->p < cts->tokens->size(cts->tokens))
+ {
+ cts->p++;
+ cts->p = skipOffTokenChannels(cts, cts->p);
+ }
+}
+
+
+/// As per ordinary consume but notifies the debugger about hidden
+/// tokens and so on.
+///
+static void
+dbgConsume (pANTLR3_INT_STREAM is)
+{
+ pANTLR3_TOKEN_STREAM ts;
+ ANTLR3_MARKER a;
+ ANTLR3_MARKER b;
+ pANTLR3_COMMON_TOKEN t;
+
+ ts = (pANTLR3_TOKEN_STREAM) is->super;
+
+ if (ts->initialStreamState == ANTLR3_TRUE)
+ {
+ consumeInitialHiddenTokens(is);
+ }
+
+ a = is->index(is); // Where are we right now?
+ t = ts->_LT(ts, 1); // Current token from stream
+
+ consume(is); // Standard consumer
+
+ b = is->index(is); // Where are we after consuming 1 on channel token?
+
+ ts->debugger->consumeToken(ts->debugger, t); // Tell the debugger that we consumed the first token
+
+ if (b>a+1)
+ {
+ // The standard consume caused the index to advance by more than 1,
+ // which can only happen if it skipped some off-channel tokens.
+ // we need to tell the debugger about those tokens.
+ //
+ ANTLR3_MARKER i;
+
+ for (i = a+1; i<b; i++)
+ {
+ ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, (ANTLR3_UINT32)i));
+ }
+
+ }
+}
+
+/** A simple filter mechanism whereby you can tell this token stream
+ * to force all tokens of type ttype to be on channel. For example,
+ * when interpreting, we cannot execute actions so we need to tell
+ * the stream to force all WS and NEWLINE to be a different, ignored,
+ * channel.
+ */
+static void
+setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel)
+{
+ if (tokenStream->channelOverrides == NULL)
+ {
+ tokenStream->channelOverrides = antlr3ListNew(10);
+ }
+
+ /* We add one to the channel so we can distinguish NULL as being no entry in the
+ * table for a particular token type.
+ */
+ tokenStream->channelOverrides->put(tokenStream->channelOverrides, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)channel + 1), NULL);
+}
+
+static void
+discardTokenType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 ttype)
+{
+ if (tokenStream->discardSet == NULL)
+ {
+ tokenStream->discardSet = antlr3ListNew(31);
+ }
+
+ /* We add one to the channel so we can distinguish NULL as being no entry in the
+ * table for a particular token type. We could use bitsets for this I suppose too.
+ */
+ tokenStream->discardSet->put(tokenStream->discardSet, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)ttype + 1), NULL);
+}
+
+static void
+discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_BOOLEAN discard)
+{
+ tokenStream->discardOffChannel = discard;
+}
+
+static pANTLR3_VECTOR
+getTokens (pANTLR3_COMMON_TOKEN_STREAM tokenStream)
+{
+ if (tokenStream->p == -1)
+ {
+ fillBuffer(tokenStream);
+ }
+
+ return tokenStream->tokens;
+}
+
+static pANTLR3_LIST
+getTokenRange (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
+{
+ return tokenStream->getTokensSet(tokenStream, start, stop, NULL);
+}
+/** Given a start and stop index, return a List of all tokens in
+ * the token type BitSet. Return null if no tokens were found. This
+ * method looks at both on and off channel tokens.
+ */
+static pANTLR3_LIST
+getTokensSet (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types)
+{
+ pANTLR3_LIST filteredList;
+ ANTLR3_UINT32 i;
+ ANTLR3_UINT32 n;
+ pANTLR3_COMMON_TOKEN tok;
+
+ if (tokenStream->p == -1)
+ {
+ fillBuffer(tokenStream);
+ }
+ if (stop > tokenStream->tstream->istream->size(tokenStream->tstream->istream))
+ {
+ stop = tokenStream->tstream->istream->size(tokenStream->tstream->istream);
+ }
+ if (start > stop)
+ {
+ return NULL;
+ }
+
+ /* We have the range set, now we need to iterate through the
+ * installed tokens and create a new list with just the ones we want
+ * in it. We are just moving pointers about really.
+ */
+ filteredList = antlr3ListNew((ANTLR3_UINT32)tokenStream->tstream->istream->size(tokenStream->tstream->istream));
+
+ for (i = start, n = 0; i<= stop; i++)
+ {
+ tok = tokenStream->tstream->get(tokenStream->tstream, i);
+
+ if ( types == NULL
+ || types->isMember(types, tok->getType(tok) == ANTLR3_TRUE)
+ )
+ {
+ filteredList->put(filteredList, n++, (void *)tok, NULL);
+ }
+ }
+
+ /* Did we get any then?
+ */
+ if (filteredList->size(filteredList) == 0)
+ {
+ filteredList->free(filteredList);
+ filteredList = NULL;
+ }
+
+ return filteredList;
+}
+
+static pANTLR3_LIST
+getTokensList (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list)
+{
+ pANTLR3_BITSET bitSet;
+ pANTLR3_LIST newlist;
+
+ bitSet = antlr3BitsetList(list->table);
+
+ newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);
+
+ bitSet->free(bitSet);
+
+ return newlist;
+
+}
+
+static pANTLR3_LIST
+getTokensType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type)
+{
+ pANTLR3_BITSET bitSet;
+ pANTLR3_LIST newlist;
+
+ bitSet = antlr3BitsetOf(type, -1);
+ newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);
+
+ bitSet->free(bitSet);
+
+ return newlist;
+}
+
+static ANTLR3_UINT32
+_LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
+{
+ pANTLR3_TOKEN_STREAM ts;
+ pANTLR3_COMMON_TOKEN tok;
+
+ ts = (pANTLR3_TOKEN_STREAM) is->super;
+
+ tok = ts->_LT(ts, i);
+
+ if (tok != NULL)
+ {
+ return tok->getType(tok);
+ }
+ else
+ {
+ return ANTLR3_TOKEN_INVALID;
+ }
+}
+
+/// As per _LA() but for debug mode.
+///
+static ANTLR3_UINT32
+dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
+{
+ pANTLR3_TOKEN_STREAM ts;
+
+ ts = (pANTLR3_TOKEN_STREAM) is->super;
+
+ if (ts->initialStreamState == ANTLR3_TRUE)
+ {
+ consumeInitialHiddenTokens(is);
+ }
+ ts->debugger->LT(ts->debugger, i, tokLT(ts, i));
+ return _LA(is, i);
+}
+
+static ANTLR3_MARKER
+mark (pANTLR3_INT_STREAM is)
+{
+ is->lastMarker = is->index(is);
+ return is->lastMarker;
+}
+
+/// As per mark() but with a call to tell the debugger we are doing this
+///
+static ANTLR3_MARKER
+dbgMark (pANTLR3_INT_STREAM is)
+{
+ pANTLR3_TOKEN_STREAM ts;
+
+ ts = (pANTLR3_TOKEN_STREAM) is->super;
+
+ is->lastMarker = is->index(is);
+ ts->debugger->mark(ts->debugger, is->lastMarker);
+
+ return is->lastMarker;
+}
+
+static void
+release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark)
+{
+ return;
+}
+
+static ANTLR3_UINT32
+size (pANTLR3_INT_STREAM is)
+{
+ pANTLR3_COMMON_TOKEN_STREAM cts;
+ pANTLR3_TOKEN_STREAM ts;
+
+ if (is->cachedSize > 0)
+ {
+ return is->cachedSize;
+ }
+ ts = (pANTLR3_TOKEN_STREAM) is->super;
+ cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
+
+ is->cachedSize = cts->tokens->count;
+ return is->cachedSize;
+}
+
+static ANTLR3_MARKER
+tindex (pANTLR3_INT_STREAM is)
+{
+ pANTLR3_COMMON_TOKEN_STREAM cts;
+ pANTLR3_TOKEN_STREAM ts;
+
+ ts = (pANTLR3_TOKEN_STREAM) is->super;
+ cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
+
+ return cts->p;
+}
+
+static void
+dbgRewindLast (pANTLR3_INT_STREAM is)
+{
+ pANTLR3_TOKEN_STREAM ts;
+
+ ts = (pANTLR3_TOKEN_STREAM) is->super;
+
+ ts->debugger->rewindLast(ts->debugger);
+
+ is->rewind(is, is->lastMarker);
+}
+static void
+rewindLast (pANTLR3_INT_STREAM is)
+{
+ is->rewind(is, is->lastMarker);
+}
+static void
+rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
+{
+ is->seek(is, (ANTLR3_UINT32)(marker));
+}
+static void
+dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
+{
+ pANTLR3_TOKEN_STREAM ts;
+
+ ts = (pANTLR3_TOKEN_STREAM) is->super;
+
+ ts->debugger->rewind(ts->debugger, marker);
+
+ is->seek(is, (ANTLR3_UINT32)(marker));
+}
+
+static void
+seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
+{
+ pANTLR3_COMMON_TOKEN_STREAM cts;
+ pANTLR3_TOKEN_STREAM ts;
+
+ ts = (pANTLR3_TOKEN_STREAM) is->super;
+ cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
+
+ cts->p = (ANTLR3_UINT32)index;
+}
+static void
+dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
+{
+ // TODO: Implement seek in debugger when Ter adds it to Java
+ //
+ seek(is, index);
+}
+ANTLR3_API void
+fillBufferExt(pANTLR3_COMMON_TOKEN_STREAM tokenStream)
+{
+ fillBuffer(tokenStream);
+}
+static void
+fillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream) {
+ ANTLR3_UINT32 index;
+ pANTLR3_COMMON_TOKEN tok;
+ ANTLR3_BOOLEAN discard;
+ void * channelI;
+
+ /* Start at index 0 of course
+ */
+ index = 0;
+
+ /* Pick out the next token from the token source
+ * Remember we just get a pointer (reference if you like) here
+ * and so if we store it anywhere, we don't set any pointers to auto free it.
+ */
+ tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);
+
+ while (tok != NULL && tok->type != ANTLR3_TOKEN_EOF)
+ {
+ discard = ANTLR3_FALSE; /* Assume we are not discarding */
+
+ /* I employ a bit of a trick, or perhaps hack here. Rather than
+ * store a pointer to a structure in the override map and discard set
+ * we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0
+ * we can distinguish "not being there" from "being channel or type 0"
+ */
+
+ if (tokenStream->discardSet != NULL
+ && tokenStream->discardSet->get(tokenStream->discardSet, tok->getType(tok)) != NULL)
+ {
+ discard = ANTLR3_TRUE;
+ }
+ else if ( tokenStream->discardOffChannel == ANTLR3_TRUE
+ && tok->getChannel(tok) != tokenStream->channel
+ )
+ {
+ discard = ANTLR3_TRUE;
+ }
+ else if (tokenStream->channelOverrides != NULL)
+ {
+ /* See if this type is in the override map
+ */
+ channelI = tokenStream->channelOverrides->get(tokenStream->channelOverrides, tok->getType(tok) + 1);
+
+ if (channelI != NULL)
+ {
+ /* Override found
+ */
+ tok->setChannel(tok, ANTLR3_UINT32_CAST(channelI) - 1);
+ }
+ }
+
+ /* If not discarding it, add it to the list at the current index
+ */
+ if (discard == ANTLR3_FALSE)
+ {
+ /* Add it, indicating that we will delete it and the table should not
+ */
+ tok->setTokenIndex(tok, index);
+ tokenStream->p++;
+ tokenStream->tokens->add(tokenStream->tokens, (void *) tok, NULL);
+ index++;
+ }
+
+ tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);
+ }
+
+ /* Cache the size so we don't keep doing indirect method calls. We do this as
+ * early as possible so that anything after this may utilize the cached value.
+ */
+ tokenStream->tstream->istream->cachedSize = tokenStream->tokens->count;
+
+ /* Set the consume pointer to the first token that is on our channel
+ */
+ tokenStream->p = 0;
+ tokenStream->p = skipOffTokenChannels(tokenStream, tokenStream->p);
+
+}
+
+/// Given a starting index, return the index of the first on-channel
+/// token.
+///
+static ANTLR3_UINT32
+skipOffTokenChannels(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i) {
+ ANTLR3_INT32 n;
+ pANTLR3_COMMON_TOKEN tok;
+
+ n = tokenStream->tstream->istream->cachedSize;
+
+ while (i < n)
+ {
+ tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[i].element;
+
+ if (tok->channel!= tokenStream->channel)
+ {
+ i++;
+ }
+ else
+ {
+ return i;
+ }
+ }
+ return i;
+}
+
+static ANTLR3_UINT32
+skipOffTokenChannelsReverse(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 x)
+{
+ pANTLR3_COMMON_TOKEN tok;
+
+ while (x >= 0)
+ {
+ tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[x].element;
+
+ if ((tok->channel != tokenStream->channel))
+ {
+ x--;
+ }
+ else
+ {
+ return x;
+ }
+ }
+ return x;
+}
+
+/// Return a string that represents the name assoicated with the input source
+///
+/// /param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream.
+///
+/// /returns
+/// /implements ANTLR3_INT_STREAM_struct::getSourceName()
+///
+static pANTLR3_STRING
+getSourceName (pANTLR3_INT_STREAM is)
+{
+ // Slightly convoluted as we must trace back to the lexer's input source
+ // via the token source. The streamName that is here is not initialized
+ // because this is a token stream, not a file or string stream, which are the
+ // only things that have a context for a source name.
+ //
+ return ((pANTLR3_TOKEN_STREAM)(is->super))->tokenSource->fileName;
+}
Code Review / simantics / platform.git / blobdiff