--- /dev/null
+/// \file \r
+/// Default implementation of CommonTokenStream\r
+///\r
+\r
+// [The "BSD licence"]\r
+// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC\r
+// http://www.temporal-wave.com\r
+// http://www.linkedin.com/in/jimidle\r
+//\r
+// All rights reserved.\r
+//\r
+// Redistribution and use in source and binary forms, with or without\r
+// modification, are permitted provided that the following conditions\r
+// are met:\r
+// 1. Redistributions of source code must retain the above copyright\r
+// notice, this list of conditions and the following disclaimer.\r
+// 2. Redistributions in binary form must reproduce the above copyright\r
+// notice, this list of conditions and the following disclaimer in the\r
+// documentation and/or other materials provided with the distribution.\r
+// 3. The name of the author may not be used to endorse or promote products\r
+// derived from this software without specific prior written permission.\r
+//\r
+// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\r
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\r
+// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\r
+// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\r
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\r
+// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\r
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\r
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\r
+// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r
+\r
+#include <antlr3tokenstream.h>\r
+\r
+#ifdef ANTLR3_WINDOWS\r
+#pragma warning( disable : 4100 )\r
+#endif\r
+\r
+// COMMON_TOKEN_STREAM API\r
+//\r
+static void setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel);\r
+static void discardTokenType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 ttype);\r
+static void discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_BOOLEAN discard);\r
+static pANTLR3_VECTOR getTokens (pANTLR3_COMMON_TOKEN_STREAM cts);\r
+static pANTLR3_LIST getTokenRange (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);\r
+static pANTLR3_LIST getTokensSet (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types);\r
+static pANTLR3_LIST getTokensList (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list);\r
+static pANTLR3_LIST getTokensType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type);\r
+\r
+// TOKEN_STREAM API \r
+//\r
+static pANTLR3_COMMON_TOKEN tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);\r
+static pANTLR3_COMMON_TOKEN dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);\r
+static pANTLR3_COMMON_TOKEN get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i);\r
+static pANTLR3_TOKEN_SOURCE getTokenSource (pANTLR3_TOKEN_STREAM ts);\r
+static void setTokenSource (pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource);\r
+static pANTLR3_STRING toString (pANTLR3_TOKEN_STREAM ts);\r
+static pANTLR3_STRING toStringSS (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);\r
+static pANTLR3_STRING toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop);\r
+static void setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger);\r
+\r
+// INT STREAM API\r
+//\r
+static void consume (pANTLR3_INT_STREAM is);\r
+static void dbgConsume (pANTLR3_INT_STREAM is);\r
+static ANTLR3_UINT32 _LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i);\r
+static ANTLR3_UINT32 dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i);\r
+static ANTLR3_MARKER mark (pANTLR3_INT_STREAM is);\r
+static ANTLR3_MARKER dbgMark (pANTLR3_INT_STREAM is);\r
+static void release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark);\r
+static ANTLR3_UINT32 size (pANTLR3_INT_STREAM is);\r
+static ANTLR3_MARKER tindex (pANTLR3_INT_STREAM is);\r
+static void rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);\r
+static void dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);\r
+static void rewindLast (pANTLR3_INT_STREAM is);\r
+static void dbgRewindLast (pANTLR3_INT_STREAM is);\r
+static void seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index);\r
+static void dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index);\r
+static pANTLR3_STRING getSourceName (pANTLR3_INT_STREAM is);\r
+static void antlr3TokenStreamFree (pANTLR3_TOKEN_STREAM stream);\r
+static void antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream);\r
+\r
+// Helpers\r
+//\r
+static void fillBuffer (pANTLR3_COMMON_TOKEN_STREAM tokenStream);\r
+static ANTLR3_UINT32 skipOffTokenChannels (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);\r
+static ANTLR3_UINT32 skipOffTokenChannelsReverse (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);\r
+static pANTLR3_COMMON_TOKEN LB (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);\r
+\r
+ANTLR3_API pANTLR3_TOKEN_STREAM\r
+antlr3TokenStreamNew()\r
+{\r
+ pANTLR3_TOKEN_STREAM stream;\r
+\r
+ // Memory for the interface structure\r
+ //\r
+ stream = (pANTLR3_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_STREAM));\r
+\r
+ if (stream == NULL)\r
+ {\r
+ return NULL;\r
+ }\r
+\r
+ // Install basic API \r
+ //\r
+ stream->free = antlr3TokenStreamFree;\r
+\r
+ \r
+ return stream;\r
+}\r
+\r
+static void\r
+antlr3TokenStreamFree(pANTLR3_TOKEN_STREAM stream)\r
+{ \r
+ ANTLR3_FREE(stream);\r
+}\r
+\r
+static void \r
+antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream)\r
+{\r
+ // We only free up our subordinate interfaces if they belong\r
+ // to us, otherwise we let whoever owns them deal with them.\r
+ //\r
+ if (stream->tstream->super == stream)\r
+ {\r
+ if (stream->tstream->istream->super == stream->tstream)\r
+ {\r
+ stream->tstream->istream->free(stream->tstream->istream);\r
+ stream->tstream->istream = NULL;\r
+ }\r
+ stream->tstream->free(stream->tstream);\r
+ }\r
+\r
+ // Now we free our own resources\r
+ //\r
+ if (stream->tokens != NULL)\r
+ {\r
+ stream->tokens->free(stream->tokens);\r
+ stream->tokens = NULL;\r
+ }\r
+ if (stream->discardSet != NULL)\r
+ {\r
+ stream->discardSet->free(stream->discardSet);\r
+ stream->discardSet = NULL;\r
+ }\r
+ if (stream->channelOverrides != NULL)\r
+ {\r
+ stream->channelOverrides->free(stream->channelOverrides);\r
+ stream->channelOverrides = NULL;\r
+ }\r
+\r
+ // Free our memory now\r
+ //\r
+ ANTLR3_FREE(stream);\r
+}\r
+\r
+ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM\r
+antlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger)\r
+{\r
+ pANTLR3_COMMON_TOKEN_STREAM stream;\r
+\r
+ // Create a standard token stream\r
+ //\r
+ stream = antlr3CommonTokenStreamSourceNew(hint, source);\r
+\r
+ // Install the debugger object\r
+ //\r
+ stream->tstream->debugger = debugger;\r
+\r
+ // Override standard token stream methods with debugging versions\r
+ //\r
+ stream->tstream->initialStreamState = ANTLR3_FALSE;\r
+\r
+ stream->tstream->_LT = dbgTokLT;\r
+\r
+ stream->tstream->istream->consume = dbgConsume;\r
+ stream->tstream->istream->_LA = dbgLA;\r
+ stream->tstream->istream->mark = dbgMark;\r
+ stream->tstream->istream->rewind = dbgRewindStream;\r
+ stream->tstream->istream->rewindLast = dbgRewindLast;\r
+ stream->tstream->istream->seek = dbgSeek;\r
+\r
+ return stream;\r
+}\r
+\r
+ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM\r
+antlr3CommonTokenStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source)\r
+{\r
+ pANTLR3_COMMON_TOKEN_STREAM stream;\r
+\r
+ stream = antlr3CommonTokenStreamNew(hint);\r
+\r
+ stream->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;\r
+ \r
+ stream->channelOverrides = NULL;\r
+ stream->discardSet = NULL;\r
+ stream->discardOffChannel = ANTLR3_FALSE;\r
+\r
+ stream->tstream->setTokenSource(stream->tstream, source);\r
+\r
+ stream->free = antlr3CTSFree;\r
+ return stream;\r
+}\r
+\r
+ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM\r
+antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint)\r
+{\r
+ pANTLR3_COMMON_TOKEN_STREAM stream;\r
+\r
+ /* Memory for the interface structure\r
+ */\r
+ stream = (pANTLR3_COMMON_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TOKEN_STREAM));\r
+\r
+ if (stream == NULL)\r
+ {\r
+ return NULL;\r
+ }\r
+\r
+ /* Create space for the token stream interface\r
+ */\r
+ stream->tstream = antlr3TokenStreamNew();\r
+ stream->tstream->super = stream;\r
+\r
+ /* Create space for the INT_STREAM interfacce\r
+ */\r
+ stream->tstream->istream = antlr3IntStreamNew();\r
+ stream->tstream->istream->super = (stream->tstream);\r
+ stream->tstream->istream->type = ANTLR3_TOKENSTREAM;\r
+\r
+ /* Install the token tracking tables\r
+ */\r
+ stream->tokens = antlr3VectorNew(0);\r
+\r
+ /* Defaults\r
+ */\r
+ stream->p = -1;\r
+\r
+ /* Install the common token stream API\r
+ */\r
+ stream->setTokenTypeChannel = setTokenTypeChannel;\r
+ stream->discardTokenType = discardTokenType;\r
+ stream->discardOffChannelToks = discardOffChannel;\r
+ stream->getTokens = getTokens;\r
+ stream->getTokenRange = getTokenRange;\r
+ stream->getTokensSet = getTokensSet;\r
+ stream->getTokensList = getTokensList;\r
+ stream->getTokensType = getTokensType;\r
+\r
+ /* Install the token stream API\r
+ */\r
+ stream->tstream->_LT = tokLT;\r
+ stream->tstream->get = get;\r
+ stream->tstream->getTokenSource = getTokenSource;\r
+ stream->tstream->setTokenSource = setTokenSource;\r
+ stream->tstream->toString = toString;\r
+ stream->tstream->toStringSS = toStringSS;\r
+ stream->tstream->toStringTT = toStringTT;\r
+ stream->tstream->setDebugListener = setDebugListener;\r
+\r
+ /* Install INT_STREAM interface\r
+ */\r
+ stream->tstream->istream->_LA = _LA;\r
+ stream->tstream->istream->mark = mark;\r
+ stream->tstream->istream->release = release;\r
+ stream->tstream->istream->size = size;\r
+ stream->tstream->istream->index = tindex;\r
+ stream->tstream->istream->rewind = rewindStream;\r
+ stream->tstream->istream->rewindLast= rewindLast;\r
+ stream->tstream->istream->seek = seek;\r
+ stream->tstream->istream->consume = consume;\r
+ stream->tstream->istream->getSourceName = getSourceName;\r
+\r
+ return stream;\r
+}\r
+\r
+// Install a debug listener adn switch to debug mode methods\r
+//\r
+static void \r
+setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger)\r
+{\r
+ // Install the debugger object\r
+ //\r
+ ts->debugger = debugger;\r
+\r
+ // Override standard token stream methods with debugging versions\r
+ //\r
+ ts->initialStreamState = ANTLR3_FALSE;\r
+\r
+ ts->_LT = dbgTokLT;\r
+\r
+ ts->istream->consume = dbgConsume;\r
+ ts->istream->_LA = dbgLA;\r
+ ts->istream->mark = dbgMark;\r
+ ts->istream->rewind = dbgRewindStream;\r
+ ts->istream->rewindLast = dbgRewindLast;\r
+ ts->istream->seek = dbgSeek;\r
+}\r
+\r
+/** Get the ith token from the current position 1..n where k=1 is the\r
+* first symbol of lookahead.\r
+*/\r
+static pANTLR3_COMMON_TOKEN \r
+tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)\r
+{\r
+ ANTLR3_INT32 i;\r
+ ANTLR3_INT32 n;\r
+ pANTLR3_COMMON_TOKEN_STREAM cts;\r
+\r
+ cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;\r
+\r
+ if (k < 0)\r
+ {\r
+ return LB(cts, -k);\r
+ }\r
+\r
+ if (cts->p == -1)\r
+ {\r
+ fillBuffer(cts);\r
+ }\r
+ if (k == 0)\r
+ {\r
+ return NULL;\r
+ }\r
+\r
+ if ((cts->p + k - 1) >= (ANTLR3_INT32)ts->istream->cachedSize)\r
+ {\r
+ pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken);\r
+\r
+ teof->setStartIndex (teof, ts->istream->index (ts->istream));\r
+ teof->setStopIndex (teof, ts->istream->index (ts->istream));\r
+ return teof;\r
+ }\r
+\r
+ i = cts->p;\r
+ n = 1;\r
+\r
+ /* Need to find k good tokens, skipping ones that are off channel\r
+ */\r
+ while ( n < k)\r
+ {\r
+ /* Skip off-channel tokens */\r
+ i = skipOffTokenChannels(cts, i+1); /* leave p on valid token */\r
+ n++;\r
+ }\r
+ if ( (ANTLR3_UINT32) i >= ts->istream->cachedSize)\r
+ {\r
+ pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken);\r
+\r
+ teof->setStartIndex (teof, ts->istream->index(ts->istream));\r
+ teof->setStopIndex (teof, ts->istream->index(ts->istream));\r
+ return teof;\r
+ }\r
+\r
+ // Here the token must be in the input vector. Rather then incut\r
+ // function call penalty, we jsut return the pointer directly\r
+ // from the vector\r
+ //\r
+ return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;\r
+ //return (pANTLR3_COMMON_TOKEN)cts->tokens->get(cts->tokens, i);\r
+}\r
+\r
+/// Debug only method to flag consumption of initial off-channel\r
+/// tokens in the input stream\r
+///\r
+static void\r
+consumeInitialHiddenTokens(pANTLR3_INT_STREAM is)\r
+{\r
+ ANTLR3_MARKER first;\r
+ ANTLR3_INT32 i;\r
+ pANTLR3_TOKEN_STREAM ts;\r
+\r
+ ts = (pANTLR3_TOKEN_STREAM) is->super;\r
+ first = is->index(is);\r
+\r
+ for (i=0; i<first; i++)\r
+ {\r
+ ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, i));\r
+ }\r
+\r
+ ts->initialStreamState = ANTLR3_FALSE;\r
+\r
+}\r
+\r
+/// As per the normal tokLT but sends information to the debugger\r
+///\r
+static pANTLR3_COMMON_TOKEN \r
+dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)\r
+{\r
+ if (ts->initialStreamState == ANTLR3_TRUE)\r
+ {\r
+ consumeInitialHiddenTokens(ts->istream);\r
+ }\r
+ return tokLT(ts, k);\r
+}\r
+\r
+#ifdef ANTLR3_WINDOWS\r
+ /* When fully optimized VC7 complains about non reachable code.\r
+ * Not yet sure if this is an optimizer bug, or a bug in the flow analysis\r
+ */\r
+#pragma warning( disable : 4702 )\r
+#endif\r
+\r
+static pANTLR3_COMMON_TOKEN\r
+LB(pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 k)\r
+{\r
+ ANTLR3_INT32 i;\r
+ ANTLR3_INT32 n;\r
+\r
+ if (cts->p == -1)\r
+ {\r
+ fillBuffer(cts);\r
+ }\r
+ if (k == 0)\r
+ {\r
+ return NULL;\r
+ }\r
+ if ((cts->p - k) < 0)\r
+ {\r
+ return NULL;\r
+ }\r
+\r
+ i = cts->p;\r
+ n = 1;\r
+\r
+ /* Need to find k good tokens, going backwards, skipping ones that are off channel\r
+ */\r
+ while (n <= (ANTLR3_INT32) k)\r
+ {\r
+ /* Skip off-channel tokens\r
+ */\r
+\r
+ i = skipOffTokenChannelsReverse(cts, i - 1); /* leave p on valid token */\r
+ n++;\r
+ }\r
+ if (i < 0)\r
+ {\r
+ return NULL;\r
+ }\r
+ // Here the token must be in the input vector. Rather then incut\r
+ // function call penalty, we jsut return the pointer directly\r
+ // from the vector\r
+ //\r
+ return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;\r
+}\r
+\r
+static pANTLR3_COMMON_TOKEN \r
+get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i)\r
+{\r
+ pANTLR3_COMMON_TOKEN_STREAM cts;\r
+\r
+ cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;\r
+\r
+ return (pANTLR3_COMMON_TOKEN)(cts->tokens->get(cts->tokens, i)); /* Token index is zero based but vectors are 1 based */\r
+}\r
+\r
+static pANTLR3_TOKEN_SOURCE \r
+getTokenSource (pANTLR3_TOKEN_STREAM ts)\r
+{\r
+ return ts->tokenSource;\r
+}\r
+\r
+static void\r
+setTokenSource ( pANTLR3_TOKEN_STREAM ts,\r
+ pANTLR3_TOKEN_SOURCE tokenSource)\r
+{\r
+ ts->tokenSource = tokenSource;\r
+}\r
+\r
+static pANTLR3_STRING \r
+toString (pANTLR3_TOKEN_STREAM ts)\r
+{\r
+ pANTLR3_COMMON_TOKEN_STREAM cts;\r
+\r
+ cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;\r
+\r
+ if (cts->p == -1)\r
+ {\r
+ fillBuffer(cts);\r
+ }\r
+\r
+ return ts->toStringSS(ts, 0, ts->istream->size(ts->istream));\r
+}\r
+\r
+static pANTLR3_STRING\r
+toStringSS(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)\r
+{\r
+ pANTLR3_STRING string;\r
+ pANTLR3_TOKEN_SOURCE tsource;\r
+ pANTLR3_COMMON_TOKEN tok;\r
+ ANTLR3_UINT32 i;\r
+ pANTLR3_COMMON_TOKEN_STREAM cts;\r
+\r
+ cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;\r
+\r
+ if (cts->p == -1)\r
+ {\r
+ fillBuffer(cts);\r
+ }\r
+ if (stop >= ts->istream->size(ts->istream))\r
+ {\r
+ stop = ts->istream->size(ts->istream) - 1;\r
+ }\r
+\r
+ /* Who is giving us these tokens?\r
+ */\r
+ tsource = ts->getTokenSource(ts);\r
+\r
+ if (tsource != NULL && cts->tokens != NULL)\r
+ {\r
+ /* Finally, let's get a string\r
+ */\r
+ string = tsource->strFactory->newRaw(tsource->strFactory);\r
+\r
+ for (i = start; i <= stop; i++)\r
+ {\r
+ tok = ts->get(ts, i);\r
+ if (tok != NULL)\r
+ {\r
+ string->appendS(string, tok->getText(tok));\r
+ }\r
+ }\r
+\r
+ return string;\r
+ }\r
+ return NULL;\r
+\r
+}\r
+\r
+static pANTLR3_STRING \r
+toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop)\r
+{\r
+ if (start != NULL && stop != NULL)\r
+ {\r
+ return ts->toStringSS(ts, (ANTLR3_UINT32)start->getTokenIndex(start), (ANTLR3_UINT32)stop->getTokenIndex(stop));\r
+ }\r
+ else\r
+ {\r
+ return NULL;\r
+ }\r
+}\r
+\r
+/** Move the input pointer to the next incoming token. The stream\r
+ * must become active with LT(1) available. consume() simply\r
+ * moves the input pointer so that LT(1) points at the next\r
+ * input symbol. Consume at least one token.\r
+ *\r
+ * Walk past any token not on the channel the parser is listening to.\r
+ */\r
+static void \r
+consume (pANTLR3_INT_STREAM is)\r
+{\r
+ pANTLR3_COMMON_TOKEN_STREAM cts;\r
+ pANTLR3_TOKEN_STREAM ts;\r
+\r
+ ts = (pANTLR3_TOKEN_STREAM) is->super;\r
+ cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;\r
+\r
+ if ((ANTLR3_UINT32)cts->p < cts->tokens->size(cts->tokens))\r
+ {\r
+ cts->p++;\r
+ cts->p = skipOffTokenChannels(cts, cts->p);\r
+ }\r
+}\r
+\r
+\r
+/// As per ordinary consume but notifies the debugger about hidden\r
+/// tokens and so on.\r
+///\r
+static void\r
+dbgConsume (pANTLR3_INT_STREAM is)\r
+{\r
+ pANTLR3_TOKEN_STREAM ts;\r
+ ANTLR3_MARKER a;\r
+ ANTLR3_MARKER b;\r
+ pANTLR3_COMMON_TOKEN t;\r
+\r
+ ts = (pANTLR3_TOKEN_STREAM) is->super;\r
+\r
+ if (ts->initialStreamState == ANTLR3_TRUE)\r
+ {\r
+ consumeInitialHiddenTokens(is);\r
+ }\r
+ \r
+ a = is->index(is); // Where are we right now?\r
+ t = ts->_LT(ts, 1); // Current token from stream\r
+\r
+ consume(is); // Standard consumer\r
+\r
+ b = is->index(is); // Where are we after consuming 1 on channel token?\r
+\r
+ ts->debugger->consumeToken(ts->debugger, t); // Tell the debugger that we consumed the first token\r
+\r
+ if (b>a+1)\r
+ {\r
+ // The standard consume caused the index to advance by more than 1,\r
+ // which can only happen if it skipped some off-channel tokens.\r
+ // we need to tell the debugger about those tokens.\r
+ //\r
+ ANTLR3_MARKER i;\r
+\r
+ for (i = a+1; i<b; i++)\r
+ {\r
+ ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, (ANTLR3_UINT32)i));\r
+ }\r
+\r
+ }\r
+}\r
+\r
+/** A simple filter mechanism whereby you can tell this token stream\r
+ * to force all tokens of type ttype to be on channel. For example,\r
+ * when interpreting, we cannot execute actions so we need to tell\r
+ * the stream to force all WS and NEWLINE to be a different, ignored,\r
+ * channel.\r
+ */\r
+static void \r
+setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel)\r
+{\r
+ if (tokenStream->channelOverrides == NULL)\r
+ {\r
+ tokenStream->channelOverrides = antlr3ListNew(10);\r
+ }\r
+\r
+ /* We add one to the channel so we can distinguish NULL as being no entry in the\r
+ * table for a particular token type.\r
+ */\r
+ tokenStream->channelOverrides->put(tokenStream->channelOverrides, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)channel + 1), NULL);\r
+}\r
+\r
+static void \r
+discardTokenType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 ttype)\r
+{\r
+ if (tokenStream->discardSet == NULL)\r
+ {\r
+ tokenStream->discardSet = antlr3ListNew(31);\r
+ }\r
+\r
+ /* We add one to the channel so we can distinguish NULL as being no entry in the\r
+ * table for a particular token type. We could use bitsets for this I suppose too.\r
+ */\r
+ tokenStream->discardSet->put(tokenStream->discardSet, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)ttype + 1), NULL);\r
+}\r
+\r
+static void \r
+discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_BOOLEAN discard)\r
+{\r
+ tokenStream->discardOffChannel = discard;\r
+}\r
+\r
+static pANTLR3_VECTOR \r
+getTokens (pANTLR3_COMMON_TOKEN_STREAM tokenStream)\r
+{\r
+ if (tokenStream->p == -1)\r
+ {\r
+ fillBuffer(tokenStream);\r
+ }\r
+\r
+ return tokenStream->tokens;\r
+}\r
+\r
+static pANTLR3_LIST \r
+getTokenRange (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)\r
+{\r
+ return tokenStream->getTokensSet(tokenStream, start, stop, NULL);\r
+} \r
+/** Given a start and stop index, return a List of all tokens in\r
+ * the token type BitSet. Return null if no tokens were found. This\r
+ * method looks at both on and off channel tokens.\r
+ */\r
+static pANTLR3_LIST \r
+getTokensSet (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types)\r
+{\r
+ pANTLR3_LIST filteredList;\r
+ ANTLR3_UINT32 i;\r
+ ANTLR3_UINT32 n;\r
+ pANTLR3_COMMON_TOKEN tok;\r
+\r
+ if (tokenStream->p == -1)\r
+ {\r
+ fillBuffer(tokenStream);\r
+ }\r
+ if (stop > tokenStream->tstream->istream->size(tokenStream->tstream->istream))\r
+ {\r
+ stop = tokenStream->tstream->istream->size(tokenStream->tstream->istream);\r
+ }\r
+ if (start > stop)\r
+ {\r
+ return NULL;\r
+ }\r
+\r
+ /* We have the range set, now we need to iterate through the\r
+ * installed tokens and create a new list with just the ones we want\r
+ * in it. We are just moving pointers about really.\r
+ */\r
+ filteredList = antlr3ListNew((ANTLR3_UINT32)tokenStream->tstream->istream->size(tokenStream->tstream->istream));\r
+\r
+ for (i = start, n = 0; i<= stop; i++)\r
+ {\r
+ tok = tokenStream->tstream->get(tokenStream->tstream, i);\r
+\r
+ if ( types == NULL\r
+ || types->isMember(types, tok->getType(tok) == ANTLR3_TRUE)\r
+ )\r
+ {\r
+ filteredList->put(filteredList, n++, (void *)tok, NULL);\r
+ }\r
+ }\r
+ \r
+ /* Did we get any then?\r
+ */\r
+ if (filteredList->size(filteredList) == 0)\r
+ {\r
+ filteredList->free(filteredList);\r
+ filteredList = NULL;\r
+ }\r
+\r
+ return filteredList;\r
+}\r
+\r
+static pANTLR3_LIST \r
+getTokensList (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list)\r
+{\r
+ pANTLR3_BITSET bitSet;\r
+ pANTLR3_LIST newlist;\r
+\r
+ bitSet = antlr3BitsetList(list->table);\r
+\r
+ newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);\r
+\r
+ bitSet->free(bitSet);\r
+\r
+ return newlist;\r
+\r
+}\r
+\r
+static pANTLR3_LIST \r
+getTokensType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type)\r
+{\r
+ pANTLR3_BITSET bitSet;\r
+ pANTLR3_LIST newlist;\r
+\r
+ bitSet = antlr3BitsetOf(type, -1);\r
+ newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);\r
+\r
+ bitSet->free(bitSet);\r
+\r
+ return newlist;\r
+}\r
+\r
+static ANTLR3_UINT32 \r
+_LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)\r
+{\r
+ pANTLR3_TOKEN_STREAM ts;\r
+ pANTLR3_COMMON_TOKEN tok;\r
+\r
+ ts = (pANTLR3_TOKEN_STREAM) is->super;\r
+\r
+ tok = ts->_LT(ts, i);\r
+\r
+ if (tok != NULL)\r
+ {\r
+ return tok->getType(tok);\r
+ }\r
+ else\r
+ {\r
+ return ANTLR3_TOKEN_INVALID;\r
+ }\r
+}\r
+\r
+/// As per _LA() but for debug mode.\r
+///\r
+static ANTLR3_UINT32 \r
+dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)\r
+{\r
+ pANTLR3_TOKEN_STREAM ts;\r
+ \r
+ ts = (pANTLR3_TOKEN_STREAM) is->super;\r
+\r
+ if (ts->initialStreamState == ANTLR3_TRUE)\r
+ {\r
+ consumeInitialHiddenTokens(is);\r
+ }\r
+ ts->debugger->LT(ts->debugger, i, tokLT(ts, i));\r
+ return _LA(is, i);\r
+}\r
+\r
+static ANTLR3_MARKER\r
+mark (pANTLR3_INT_STREAM is)\r
+{\r
+ is->lastMarker = is->index(is);\r
+ return is->lastMarker;\r
+}\r
+\r
+/// As per mark() but with a call to tell the debugger we are doing this\r
+///\r
+static ANTLR3_MARKER\r
+dbgMark (pANTLR3_INT_STREAM is)\r
+{\r
+ pANTLR3_TOKEN_STREAM ts;\r
+ \r
+ ts = (pANTLR3_TOKEN_STREAM) is->super;\r
+ \r
+ is->lastMarker = is->index(is);\r
+ ts->debugger->mark(ts->debugger, is->lastMarker);\r
+\r
+ return is->lastMarker;\r
+}\r
+\r
+static void \r
+release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark)\r
+{\r
+ return;\r
+}\r
+\r
+static ANTLR3_UINT32 \r
+size (pANTLR3_INT_STREAM is)\r
+{\r
+ pANTLR3_COMMON_TOKEN_STREAM cts;\r
+ pANTLR3_TOKEN_STREAM ts;\r
+\r
+ if (is->cachedSize > 0)\r
+ {\r
+ return is->cachedSize;\r
+ }\r
+ ts = (pANTLR3_TOKEN_STREAM) is->super;\r
+ cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;\r
+\r
+ is->cachedSize = cts->tokens->count;\r
+ return is->cachedSize;\r
+}\r
+\r
+static ANTLR3_MARKER \r
+tindex (pANTLR3_INT_STREAM is)\r
+{\r
+ pANTLR3_COMMON_TOKEN_STREAM cts;\r
+ pANTLR3_TOKEN_STREAM ts;\r
+\r
+ ts = (pANTLR3_TOKEN_STREAM) is->super;\r
+ cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;\r
+\r
+ return cts->p;\r
+}\r
+\r
+static void \r
+dbgRewindLast (pANTLR3_INT_STREAM is)\r
+{\r
+ pANTLR3_TOKEN_STREAM ts;\r
+\r
+ ts = (pANTLR3_TOKEN_STREAM) is->super;\r
+\r
+ ts->debugger->rewindLast(ts->debugger);\r
+\r
+ is->rewind(is, is->lastMarker);\r
+}\r
+static void \r
+rewindLast (pANTLR3_INT_STREAM is)\r
+{\r
+ is->rewind(is, is->lastMarker);\r
+}\r
+static void \r
+rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)\r
+{\r
+ is->seek(is, (ANTLR3_UINT32)(marker));\r
+}\r
+static void \r
+dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)\r
+{\r
+ pANTLR3_TOKEN_STREAM ts;\r
+\r
+ ts = (pANTLR3_TOKEN_STREAM) is->super;\r
+\r
+ ts->debugger->rewind(ts->debugger, marker);\r
+\r
+ is->seek(is, (ANTLR3_UINT32)(marker));\r
+}\r
+\r
+static void \r
+seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)\r
+{\r
+ pANTLR3_COMMON_TOKEN_STREAM cts;\r
+ pANTLR3_TOKEN_STREAM ts;\r
+\r
+ ts = (pANTLR3_TOKEN_STREAM) is->super;\r
+ cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;\r
+\r
+ cts->p = (ANTLR3_UINT32)index;\r
+}\r
+static void \r
+dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)\r
+{\r
+ // TODO: Implement seek in debugger when Ter adds it to Java\r
+ //\r
+ seek(is, index);\r
+}\r
+ANTLR3_API void\r
+fillBufferExt(pANTLR3_COMMON_TOKEN_STREAM tokenStream)\r
+{\r
+ fillBuffer(tokenStream);\r
+}\r
+static void\r
+fillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream) {\r
+ ANTLR3_UINT32 index;\r
+ pANTLR3_COMMON_TOKEN tok;\r
+ ANTLR3_BOOLEAN discard;\r
+ void * channelI;\r
+\r
+ /* Start at index 0 of course\r
+ */\r
+ index = 0;\r
+\r
+ /* Pick out the next token from the token source\r
+ * Remember we just get a pointer (reference if you like) here\r
+ * and so if we store it anywhere, we don't set any pointers to auto free it.\r
+ */\r
+ tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);\r
+\r
+ while (tok != NULL && tok->type != ANTLR3_TOKEN_EOF)\r
+ {\r
+ discard = ANTLR3_FALSE; /* Assume we are not discarding */\r
+\r
+ /* I employ a bit of a trick, or perhaps hack here. Rather than\r
+ * store a pointer to a structure in the override map and discard set\r
+ * we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0\r
+ * we can distinguish "not being there" from "being channel or type 0"\r
+ */\r
+\r
+ if (tokenStream->discardSet != NULL\r
+ && tokenStream->discardSet->get(tokenStream->discardSet, tok->getType(tok)) != NULL)\r
+ {\r
+ discard = ANTLR3_TRUE;\r
+ }\r
+ else if ( tokenStream->discardOffChannel == ANTLR3_TRUE\r
+ && tok->getChannel(tok) != tokenStream->channel\r
+ )\r
+ {\r
+ discard = ANTLR3_TRUE;\r
+ }\r
+ else if (tokenStream->channelOverrides != NULL)\r
+ {\r
+ /* See if this type is in the override map\r
+ */\r
+ channelI = tokenStream->channelOverrides->get(tokenStream->channelOverrides, tok->getType(tok) + 1);\r
+\r
+ if (channelI != NULL)\r
+ {\r
+ /* Override found\r
+ */\r
+ tok->setChannel(tok, ANTLR3_UINT32_CAST(channelI) - 1);\r
+ }\r
+ }\r
+\r
+ /* If not discarding it, add it to the list at the current index\r
+ */\r
+ if (discard == ANTLR3_FALSE)\r
+ {\r
+ /* Add it, indicating that we will delete it and the table should not\r
+ */\r
+ tok->setTokenIndex(tok, index);\r
+ tokenStream->p++;\r
+ tokenStream->tokens->add(tokenStream->tokens, (void *) tok, NULL);\r
+ index++;\r
+ }\r
+\r
+ tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);\r
+ }\r
+\r
+ /* Cache the size so we don't keep doing indirect method calls. We do this as\r
+ * early as possible so that anything after this may utilize the cached value.\r
+ */\r
+ tokenStream->tstream->istream->cachedSize = tokenStream->tokens->count;\r
+\r
+ /* Set the consume pointer to the first token that is on our channel\r
+ */\r
+ tokenStream->p = 0;\r
+ tokenStream->p = skipOffTokenChannels(tokenStream, tokenStream->p);\r
+\r
+}\r
+\r
+/// Given a starting index, return the index of the first on-channel\r
+/// token.\r
+///\r
+static ANTLR3_UINT32\r
+skipOffTokenChannels(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i) {\r
+ ANTLR3_INT32 n;\r
+ pANTLR3_COMMON_TOKEN tok;\r
+\r
+ n = tokenStream->tstream->istream->cachedSize;\r
+\r
+ while (i < n)\r
+ {\r
+ tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[i].element;\r
+\r
+ if (tok->channel!= tokenStream->channel)\r
+ {\r
+ i++;\r
+ }\r
+ else\r
+ {\r
+ return i;\r
+ }\r
+ }\r
+ return i;\r
+}\r
+\r
+static ANTLR3_UINT32\r
+skipOffTokenChannelsReverse(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 x)\r
+{\r
+ pANTLR3_COMMON_TOKEN tok;\r
+\r
+ while (x >= 0)\r
+ {\r
+ tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[x].element;\r
+ \r
+ if ((tok->channel != tokenStream->channel))\r
+ {\r
+ x--;\r
+ }\r
+ else\r
+ {\r
+ return x;\r
+ }\r
+ }\r
+ return x;\r
+}\r
+\r
+/// Return a string that represents the name assoicated with the input source\r
+///\r
+/// /param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream.\r
+///\r
+/// /returns \r
+/// /implements ANTLR3_INT_STREAM_struct::getSourceName()\r
+///\r
+static pANTLR3_STRING \r
+getSourceName (pANTLR3_INT_STREAM is)\r
+{\r
+ // Slightly convoluted as we must trace back to the lexer's input source\r
+ // via the token source. The streamName that is here is not initialized\r
+ // because this is a token stream, not a file or string stream, which are the\r
+ // only things that have a context for a source name.\r
+ //\r
+ return ((pANTLR3_TOKEN_STREAM)(is->super))->tokenSource->fileName;\r
+}\r
Code Review / simantics / platform.git / blobdiff