-/** \file\r
- * Defines the interface for an ANTLR3 common token stream. Custom token streams should create\r
- * one of these and then override any functions by installing their own pointers\r
- * to implement the various functions.\r
- */\r
-#ifndef _ANTLR3_TOKENSTREAM_H\r
-#define _ANTLR3_TOKENSTREAM_H\r
-\r
-// [The "BSD licence"]\r
-// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC\r
-// http://www.temporal-wave.com\r
-// http://www.linkedin.com/in/jimidle\r
-//\r
-// All rights reserved.\r
-//\r
-// Redistribution and use in source and binary forms, with or without\r
-// modification, are permitted provided that the following conditions\r
-// are met:\r
-// 1. Redistributions of source code must retain the above copyright\r
-// notice, this list of conditions and the following disclaimer.\r
-// 2. Redistributions in binary form must reproduce the above copyright\r
-// notice, this list of conditions and the following disclaimer in the\r
-// documentation and/or other materials provided with the distribution.\r
-// 3. The name of the author may not be used to endorse or promote products\r
-// derived from this software without specific prior written permission.\r
-//\r
-// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\r
-// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\r
-// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\r
-// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\r
-// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\r
-// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\r
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\r
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\r
-// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r
-\r
-#include <antlr3defs.h>\r
-#include <antlr3string.h>\r
-#include <antlr3collections.h>\r
-#include <antlr3input.h>\r
-#include <antlr3commontoken.h>\r
-#include <antlr3bitset.h>\r
-#include <antlr3debugeventlistener.h>\r
-\r
-#ifdef __cplusplus\r
-extern "C" {\r
-#endif\r
-\r
-/** Definition of a token source, which has a pointer to a function that \r
- * returns the next token (using a token factory if it is going to be\r
- * efficient) and a pointer to an ANTLR3_INPUT_STREAM. This is slightly\r
- * different to the Java interface because we have no way to implement\r
- * multiple interfaces without defining them in the interface structure\r
- * or casting (void *), which is too convoluted.\r
- */\r
-typedef struct ANTLR3_TOKEN_SOURCE_struct\r
-{\r
- /** Pointer to a function that returns the next token in the stream. \r
- */\r
- pANTLR3_COMMON_TOKEN (*nextToken)(struct ANTLR3_TOKEN_SOURCE_struct * tokenSource);\r
-\r
- /** Whoever is providing tokens, needs to provide a string factory too\r
- */\r
- pANTLR3_STRING_FACTORY strFactory;\r
-\r
- /** A special pre-allocated token, which signifies End Of Tokens. Because this must\r
- * be set up with the current input index and so on, we embed the structure and \r
- * return the address of it. It is marked as factoryMade, so that it is never\r
- * attempted to be freed.\r
- */\r
- ANTLR3_COMMON_TOKEN eofToken;\r
-\r
- /// A special pre-allocated token, which is returned by mTokens() if the\r
- /// lexer rule said to just skip the generated token altogether.\r
- /// Having this single token stops us wasting memory by have the token factory\r
- /// actually create something that we are going to SKIP(); anyway.\r
- ///\r
- ANTLR3_COMMON_TOKEN skipToken;\r
-\r
- /** Whatever is supplying the token source interface, needs a pointer to \r
- * itself so that this pointer can be passed to it when the nextToken\r
- * function is called.\r
- */\r
- void * super;\r
-\r
- /** When the token source is constructed, it is populated with the file\r
- * name from whence the tokens were produced by the lexer. This pointer is a\r
- * copy of the one supplied by the CharStream (and may be NULL) so should\r
- * not be manipulated other than to copy or print it.\r
- */\r
- pANTLR3_STRING fileName;\r
-}\r
- ANTLR3_TOKEN_SOURCE;\r
-\r
-/** Definition of the ANTLR3 common token stream interface.\r
- * \remark\r
- * Much of the documentation for this interface is stolen from Ter's Java implementation.\r
- */\r
-typedef struct ANTLR3_TOKEN_STREAM_struct\r
-{\r
- /** Pointer to the token source for this stream\r
- */\r
- pANTLR3_TOKEN_SOURCE tokenSource;\r
-\r
- /** Whatever is providing this interface needs a pointer to itself\r
- * so that this can be passed back to it whenever the api functions\r
- * are called.\r
- */\r
- void * super;\r
-\r
- /** All input streams implement the ANTLR3_INT_STREAM interface...\r
- */\r
- pANTLR3_INT_STREAM istream;\r
-\r
- /// Debugger interface, is this is a debugging token stream\r
- ///\r
- pANTLR3_DEBUG_EVENT_LISTENER debugger;\r
-\r
- /// Indicates the initial stream state for dbgConsume()\r
- ///\r
- ANTLR3_BOOLEAN initialStreamState;\r
-\r
- /** Get Token at current input pointer + i ahead where i=1 is next Token.\r
- * i<0 indicates tokens in the past. So -1 is previous token and -2 is\r
- * two tokens ago. LT(0) is undefined. For i>=n, return Token.EOFToken.\r
- * Return null for LT(0) and any index that results in an absolute address\r
- * that is negative.\r
- */\r
- pANTLR3_COMMON_TOKEN (*_LT) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream, ANTLR3_INT32 k);\r
-\r
- /** Get a token at an absolute index i; 0..n-1. This is really only\r
- * needed for profiling and debugging and token stream rewriting.\r
- * If you don't want to buffer up tokens, then this method makes no\r
- * sense for you. Naturally you can't use the rewrite stream feature.\r
- * I believe DebugTokenStream can easily be altered to not use\r
- * this method, removing the dependency.\r
- */\r
- pANTLR3_COMMON_TOKEN (*get) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream, ANTLR3_UINT32 i);\r
-\r
- /** Where is this stream pulling tokens from? This is not the name, but\r
- * a pointer into an interface that contains a ANTLR3_TOKEN_SOURCE interface.\r
- * The Token Source interface contains a pointer to the input stream and a pointer\r
- * to a function that returns the next token.\r
- */\r
- pANTLR3_TOKEN_SOURCE (*getTokenSource) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream);\r
-\r
- /** Function that installs a token source for teh stream\r
- */\r
- void (*setTokenSource) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream,\r
- pANTLR3_TOKEN_SOURCE tokenSource);\r
-\r
- /** Return the text of all the tokens in the stream, as the old tramp in \r
- * Leeds market used to say; "Get the lot!"\r
- */\r
- pANTLR3_STRING (*toString) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream);\r
-\r
- /** Return the text of all tokens from start to stop, inclusive.\r
- * If the stream does not buffer all the tokens then it can just\r
- * return an empty ANTLR3_STRING or NULL; Grammars should not access $ruleLabel.text in\r
- * an action in that case.\r
- */\r
- pANTLR3_STRING (*toStringSS) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);\r
-\r
- /** Because the user is not required to use a token with an index stored\r
- * in it, we must provide a means for two token objects themselves to\r
- * indicate the start/end location. Most often this will just delegate\r
- * to the other toString(int,int). This is also parallel with\r
- * the pTREENODE_STREAM->toString(Object,Object).\r
- */\r
- pANTLR3_STRING (*toStringTT) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop);\r
-\r
-\r
- /** Function that sets the token stream into debugging mode\r
- */\r
- void (*setDebugListener) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream, pANTLR3_DEBUG_EVENT_LISTENER debugger);\r
-\r
- /** Function that knows how to free the memory for an ANTLR3_TOKEN_STREAM\r
- */\r
- void (*free) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream);\r
-}\r
- ANTLR3_TOKEN_STREAM;\r
-\r
-/** Common token stream is an implementation of ANTLR_TOKEN_STREAM for the default\r
- * parsers and recognizers. You may of course build your own implementation if\r
- * you are so inclined.\r
- */\r
-typedef struct ANTLR3_COMMON_TOKEN_STREAM_struct\r
-{\r
- /** The ANTLR3_TOKEN_STREAM interface implementation, which also includes\r
- * the intstream implementation. We could duplicate the pANTLR_INT_STREAM\r
- * in this interface and initialize it to a copy, but this could be confusing\r
- * it just results in one more level of indirection and I think that with\r
- * judicial use of 'const' later, the optimizer will do decent job.\r
- */\r
- pANTLR3_TOKEN_STREAM tstream;\r
-\r
- /** Whatever is supplying the COMMON_TOKEN_STREAM needs a pointer to itself\r
- * so that this can be accessed by any of the API functions which it implements.\r
- */\r
- void * super;\r
-\r
- /** Records every single token pulled from the source indexed by the token index.\r
- * There might be more efficient ways to do this, such as referencing directly in to\r
- * the token factory pools, but for now this is convenient and the ANTLR3_LIST is not\r
- * a huge overhead as it only stores pointers anyway, but allows for iterations and \r
- * so on.\r
- */\r
- pANTLR3_VECTOR tokens;\r
-\r
- /** Override map of tokens. If a token type has an entry in here, then\r
- * the pointer in the table points to an int, being the override channel number\r
- * that should always be used for this token type.\r
- */\r
- pANTLR3_LIST channelOverrides;\r
-\r
- /** Discared set. If a token has an entry in this table, then it is thrown\r
- * away (data pointer is always NULL).\r
- */\r
- pANTLR3_LIST discardSet;\r
-\r
- /* The channel number that this token stream is tuned to. For instance, whitespace\r
- * is usually tuned to channel 99, which no token stream would normally tune to and\r
- * so it is thrown away.\r
- */\r
- ANTLR3_UINT32 channel;\r
-\r
- /** If this flag is set to ANTLR3_TRUE, then tokens that the stream sees that are not\r
- * in the channel that this stream is tuned to, are not tracked in the\r
- * tokens table. When set to false, ALL tokens are added to the tracking.\r
- */\r
- ANTLR3_BOOLEAN discardOffChannel;\r
-\r
- /** The index into the tokens list of the current token (the next one that will be\r
- * consumed. p = -1 indicates that the token list is empty.\r
- */\r
- ANTLR3_INT32 p;\r
-\r
- /** A simple filter mechanism whereby you can tell this token stream\r
- * to force all tokens of type ttype to be on channel. For example,\r
- * when interpreting, we cannot exec actions so we need to tell\r
- * the stream to force all WS and NEWLINE to be a different, ignored\r
- * channel.\r
- */\r
- void (*setTokenTypeChannel) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, \r
- ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel);\r
-\r
- /** Add a particular token type to the discard set. If a token is found to belong \r
- * to this set, then it is skipped/thrown away\r
- */\r
- void (*discardTokenType) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, ANTLR3_INT32 ttype);\r
-\r
- /** Signal to discard off channel tokens from here on in.\r
- */\r
- void (*discardOffChannelToks)(struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, ANTLR3_BOOLEAN discard);\r
-\r
- /** Function that returns a pointer to the ANTLR3_LIST of all tokens\r
- * in the stream (this causes the buffer to fill if we have not get any yet)\r
- */\r
- pANTLR3_VECTOR (*getTokens) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream);\r
-\r
- /** Function that returns all the tokens between a start and a stop index.\r
- * TODO: This is a new list (Ack! Maybe this is a reason to have factories for LISTS and HASHTABLES etc :-( come back to this)\r
- */\r
- pANTLR3_LIST (*getTokenRange) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);\r
-\r
- /** Function that returns all the tokens indicated by the specified bitset, within a range of tokens\r
- */\r
- pANTLR3_LIST (*getTokensSet) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, \r
- ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types);\r
- \r
- /** Function that returns all the tokens indicated by being a member of the supplied List\r
- */\r
- pANTLR3_LIST (*getTokensList) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, \r
- ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list);\r
-\r
- /** Function that returns all tokens of a certain type within a range.\r
- */\r
- pANTLR3_LIST (*getTokensType) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, \r
- ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type);\r
-\r
-\r
- /** Function that knows how to free an ANTLR3_COMMON_TOKEN_STREAM\r
- */\r
- void (*free) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream);\r
-}\r
- ANTLR3_COMMON_TOKEN_STREAM;\r
-\r
-#ifdef __cplusplus\r
-}\r
-#endif\r
-\r
-#endif\r
+/** \file
+ * Defines the interface for an ANTLR3 common token stream. Custom token streams should create
+ * one of these and then override any functions by installing their own pointers
+ * to implement the various functions.
+ */
+#ifndef _ANTLR3_TOKENSTREAM_H
+#define _ANTLR3_TOKENSTREAM_H
+
+// [The "BSD licence"]
+// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
+// http://www.temporal-wave.com
+// http://www.linkedin.com/in/jimidle
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. The name of the author may not be used to endorse or promote products
+// derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <antlr3defs.h>
+#include <antlr3string.h>
+#include <antlr3collections.h>
+#include <antlr3input.h>
+#include <antlr3commontoken.h>
+#include <antlr3bitset.h>
+#include <antlr3debugeventlistener.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Definition of a token source, which has a pointer to a function that
+ * returns the next token (using a token factory if it is going to be
+ * efficient) and a pointer to an ANTLR3_INPUT_STREAM. This is slightly
+ * different to the Java interface because we have no way to implement
+ * multiple interfaces without defining them in the interface structure
+ * or casting (void *), which is too convoluted.
+ */
+typedef struct ANTLR3_TOKEN_SOURCE_struct
+{
+ /** Pointer to a function that returns the next token in the stream.
+ */
+ pANTLR3_COMMON_TOKEN (*nextToken)(struct ANTLR3_TOKEN_SOURCE_struct * tokenSource);
+
+ /** Whoever is providing tokens, needs to provide a string factory too
+ */
+ pANTLR3_STRING_FACTORY strFactory;
+
+ /** A special pre-allocated token, which signifies End Of Tokens. Because this must
+ * be set up with the current input index and so on, we embed the structure and
+ * return the address of it. It is marked as factoryMade, so that it is never
+ * attempted to be freed.
+ */
+ ANTLR3_COMMON_TOKEN eofToken;
+
+ /// A special pre-allocated token, which is returned by mTokens() if the
+ /// lexer rule said to just skip the generated token altogether.
+ /// Having this single token stops us wasting memory by have the token factory
+ /// actually create something that we are going to SKIP(); anyway.
+ ///
+ ANTLR3_COMMON_TOKEN skipToken;
+
+ /** Whatever is supplying the token source interface, needs a pointer to
+ * itself so that this pointer can be passed to it when the nextToken
+ * function is called.
+ */
+ void * super;
+
+ /** When the token source is constructed, it is populated with the file
+ * name from whence the tokens were produced by the lexer. This pointer is a
+ * copy of the one supplied by the CharStream (and may be NULL) so should
+ * not be manipulated other than to copy or print it.
+ */
+ pANTLR3_STRING fileName;
+}
+ ANTLR3_TOKEN_SOURCE;
+
+/** Definition of the ANTLR3 common token stream interface.
+ * \remark
+ * Much of the documentation for this interface is stolen from Ter's Java implementation.
+ */
+typedef struct ANTLR3_TOKEN_STREAM_struct
+{
+ /** Pointer to the token source for this stream
+ */
+ pANTLR3_TOKEN_SOURCE tokenSource;
+
+ /** Whatever is providing this interface needs a pointer to itself
+ * so that this can be passed back to it whenever the api functions
+ * are called.
+ */
+ void * super;
+
+ /** All input streams implement the ANTLR3_INT_STREAM interface...
+ */
+ pANTLR3_INT_STREAM istream;
+
+ /// Debugger interface, is this is a debugging token stream
+ ///
+ pANTLR3_DEBUG_EVENT_LISTENER debugger;
+
+ /// Indicates the initial stream state for dbgConsume()
+ ///
+ ANTLR3_BOOLEAN initialStreamState;
+
+ /** Get Token at current input pointer + i ahead where i=1 is next Token.
+ * i<0 indicates tokens in the past. So -1 is previous token and -2 is
+ * two tokens ago. LT(0) is undefined. For i>=n, return Token.EOFToken.
+ * Return null for LT(0) and any index that results in an absolute address
+ * that is negative.
+ */
+ pANTLR3_COMMON_TOKEN (*_LT) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream, ANTLR3_INT32 k);
+
+ /** Get a token at an absolute index i; 0..n-1. This is really only
+ * needed for profiling and debugging and token stream rewriting.
+ * If you don't want to buffer up tokens, then this method makes no
+ * sense for you. Naturally you can't use the rewrite stream feature.
+ * I believe DebugTokenStream can easily be altered to not use
+ * this method, removing the dependency.
+ */
+ pANTLR3_COMMON_TOKEN (*get) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream, ANTLR3_UINT32 i);
+
+ /** Where is this stream pulling tokens from? This is not the name, but
+ * a pointer into an interface that contains a ANTLR3_TOKEN_SOURCE interface.
+ * The Token Source interface contains a pointer to the input stream and a pointer
+ * to a function that returns the next token.
+ */
+ pANTLR3_TOKEN_SOURCE (*getTokenSource) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream);
+
+ /** Function that installs a token source for teh stream
+ */
+ void (*setTokenSource) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream,
+ pANTLR3_TOKEN_SOURCE tokenSource);
+
+ /** Return the text of all the tokens in the stream, as the old tramp in
+ * Leeds market used to say; "Get the lot!"
+ */
+ pANTLR3_STRING (*toString) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream);
+
+ /** Return the text of all tokens from start to stop, inclusive.
+ * If the stream does not buffer all the tokens then it can just
+ * return an empty ANTLR3_STRING or NULL; Grammars should not access $ruleLabel.text in
+ * an action in that case.
+ */
+ pANTLR3_STRING (*toStringSS) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
+
+ /** Because the user is not required to use a token with an index stored
+ * in it, we must provide a means for two token objects themselves to
+ * indicate the start/end location. Most often this will just delegate
+ * to the other toString(int,int). This is also parallel with
+ * the pTREENODE_STREAM->toString(Object,Object).
+ */
+ pANTLR3_STRING (*toStringTT) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop);
+
+
+ /** Function that sets the token stream into debugging mode
+ */
+ void (*setDebugListener) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream, pANTLR3_DEBUG_EVENT_LISTENER debugger);
+
+ /** Function that knows how to free the memory for an ANTLR3_TOKEN_STREAM
+ */
+ void (*free) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream);
+}
+ ANTLR3_TOKEN_STREAM;
+
+/** Common token stream is an implementation of ANTLR_TOKEN_STREAM for the default
+ * parsers and recognizers. You may of course build your own implementation if
+ * you are so inclined.
+ */
+typedef struct ANTLR3_COMMON_TOKEN_STREAM_struct
+{
+ /** The ANTLR3_TOKEN_STREAM interface implementation, which also includes
+ * the intstream implementation. We could duplicate the pANTLR_INT_STREAM
+ * in this interface and initialize it to a copy, but this could be confusing
+ * it just results in one more level of indirection and I think that with
+ * judicial use of 'const' later, the optimizer will do decent job.
+ */
+ pANTLR3_TOKEN_STREAM tstream;
+
+ /** Whatever is supplying the COMMON_TOKEN_STREAM needs a pointer to itself
+ * so that this can be accessed by any of the API functions which it implements.
+ */
+ void * super;
+
+ /** Records every single token pulled from the source indexed by the token index.
+ * There might be more efficient ways to do this, such as referencing directly in to
+ * the token factory pools, but for now this is convenient and the ANTLR3_LIST is not
+ * a huge overhead as it only stores pointers anyway, but allows for iterations and
+ * so on.
+ */
+ pANTLR3_VECTOR tokens;
+
+ /** Override map of tokens. If a token type has an entry in here, then
+ * the pointer in the table points to an int, being the override channel number
+ * that should always be used for this token type.
+ */
+ pANTLR3_LIST channelOverrides;
+
+ /** Discared set. If a token has an entry in this table, then it is thrown
+ * away (data pointer is always NULL).
+ */
+ pANTLR3_LIST discardSet;
+
+ /* The channel number that this token stream is tuned to. For instance, whitespace
+ * is usually tuned to channel 99, which no token stream would normally tune to and
+ * so it is thrown away.
+ */
+ ANTLR3_UINT32 channel;
+
+ /** If this flag is set to ANTLR3_TRUE, then tokens that the stream sees that are not
+ * in the channel that this stream is tuned to, are not tracked in the
+ * tokens table. When set to false, ALL tokens are added to the tracking.
+ */
+ ANTLR3_BOOLEAN discardOffChannel;
+
+ /** The index into the tokens list of the current token (the next one that will be
+ * consumed. p = -1 indicates that the token list is empty.
+ */
+ ANTLR3_INT32 p;
+
+ /** A simple filter mechanism whereby you can tell this token stream
+ * to force all tokens of type ttype to be on channel. For example,
+ * when interpreting, we cannot exec actions so we need to tell
+ * the stream to force all WS and NEWLINE to be a different, ignored
+ * channel.
+ */
+ void (*setTokenTypeChannel) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream,
+ ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel);
+
+ /** Add a particular token type to the discard set. If a token is found to belong
+ * to this set, then it is skipped/thrown away
+ */
+ void (*discardTokenType) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, ANTLR3_INT32 ttype);
+
+ /** Signal to discard off channel tokens from here on in.
+ */
+ void (*discardOffChannelToks)(struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, ANTLR3_BOOLEAN discard);
+
+ /** Function that returns a pointer to the ANTLR3_LIST of all tokens
+ * in the stream (this causes the buffer to fill if we have not get any yet)
+ */
+ pANTLR3_VECTOR (*getTokens) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream);
+
+ /** Function that returns all the tokens between a start and a stop index.
+ * TODO: This is a new list (Ack! Maybe this is a reason to have factories for LISTS and HASHTABLES etc :-( come back to this)
+ */
+ pANTLR3_LIST (*getTokenRange) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
+
+ /** Function that returns all the tokens indicated by the specified bitset, within a range of tokens
+ */
+ pANTLR3_LIST (*getTokensSet) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream,
+ ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types);
+
+ /** Function that returns all the tokens indicated by being a member of the supplied List
+ */
+ pANTLR3_LIST (*getTokensList) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream,
+ ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list);
+
+ /** Function that returns all tokens of a certain type within a range.
+ */
+ pANTLR3_LIST (*getTokensType) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream,
+ ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type);
+
+
+ /** Function that knows how to free an ANTLR3_COMMON_TOKEN_STREAM
+ */
+ void (*free) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream);
+}
+ ANTLR3_COMMON_TOKEN_STREAM;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
Code Review / simantics / platform.git / blobdiff