--- /dev/null
+/** \file \r
+ * Base interface for any ANTLR3 lexer.\r
+ *\r
+ * An ANLTR3 lexer builds from two sets of components:\r
+ *\r
+ * - The runtime components that provide common functionality such as\r
+ * traversing character streams, building tokens for output and so on.\r
+ * - The generated rules and struutre of the actual lexer, which call upon the\r
+ * runtime components.\r
+ *\r
+ * A lexer class contains a character input stream, a base recognizer interface \r
+ * (which it will normally implement) and a token source interface (which it also\r
+ * implements. The Tokensource interface is called by a token consumer (such as\r
+ * a parser, but in theory it can be anything that wants a set of abstract\r
+ * tokens in place of a raw character stream.\r
+ *\r
+ * So then, we set up a lexer in a sequence akin to:\r
+ *\r
+ * - Create a character stream (something which implements ANTLR3_INPUT_STREAM)\r
+ * and initialize it.\r
+ * - Create a lexer interface and tell it where it its input stream is.\r
+ * This will cause the creation of a base recognizer class, which it will \r
+ * override with its own implementations of some methods. The lexer creator\r
+ * can also then in turn override anything it likes. \r
+ * - The lexer token source interface is then passed to some interface that\r
+ * knows how to use it, byte calling for a next token. \r
+ * - When a next token is called, let ze lexing begin.\r
+ *\r
+ */\r
+#ifndef _ANTLR3_LEXER\r
+#define _ANTLR3_LEXER\r
+\r
+// [The "BSD licence"]\r
+// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC\r
+// http://www.temporal-wave.com\r
+// http://www.linkedin.com/in/jimidle\r
+//\r
+// All rights reserved.\r
+//\r
+// Redistribution and use in source and binary forms, with or without\r
+// modification, are permitted provided that the following conditions\r
+// are met:\r
+// 1. Redistributions of source code must retain the above copyright\r
+// notice, this list of conditions and the following disclaimer.\r
+// 2. Redistributions in binary form must reproduce the above copyright\r
+// notice, this list of conditions and the following disclaimer in the\r
+// documentation and/or other materials provided with the distribution.\r
+// 3. The name of the author may not be used to endorse or promote products\r
+// derived from this software without specific prior written permission.\r
+//\r
+// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\r
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\r
+// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\r
+// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\r
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\r
+// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\r
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\r
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\r
+// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r
+\r
+/* Definitions\r
+ */\r
+#define ANTLR3_STRING_TERMINATOR 0xFFFFFFFF\r
+\r
+#include <antlr3defs.h>\r
+#include <antlr3input.h>\r
+#include <antlr3commontoken.h>\r
+#include <antlr3tokenstream.h>\r
+#include <antlr3baserecognizer.h>\r
+\r
+#ifdef __cplusplus\r
+extern "C" {\r
+#endif\r
+\r
+typedef struct ANTLR3_LEXER_struct\r
+{\r
+ /** If there is a super structure that is implementing the\r
+ * lexer, then a pointer to it can be stored here in case\r
+ * implementing functions are overridden by this super structure.\r
+ */\r
+ void * super;\r
+\r
+ /** A generated lexer has an mTokens() function, which needs\r
+ * the context pointer of the generated lexer, not the base lexer interface\r
+ * this is stored here and initialized by the generated code (or manually\r
+ * if this is a manually built lexer.\r
+ */\r
+ void * ctx;\r
+\r
+ /** A pointer to the character stream whence this lexer is receiving\r
+ * characters. \r
+ * TODO: I may come back to this and implement charstream outside\r
+ * the input stream as per the java implementation.\r
+ */\r
+ pANTLR3_INPUT_STREAM input;\r
+\r
+ /** Pointer to the implementation of a base recognizer, which the lexer\r
+ * creates and then overrides with its own lexer oriented functions (the \r
+ * default implementation is parser oriented). This also contains a\r
+ * token source interface, which the lexer instance will provide to anything \r
+ * that needs it, which is anything else that implements a base recognizer,\r
+ * such as a parser.\r
+ */\r
+ pANTLR3_BASE_RECOGNIZER rec;\r
+\r
+ /** Pointer to a function that sets the charstream source for the lexer and\r
+ * causes it to be reset.\r
+ */\r
+ void (*setCharStream) (struct ANTLR3_LEXER_struct * lexer, pANTLR3_INPUT_STREAM input);\r
+ \r
+ /** Pointer to a function that switches the current character input stream to \r
+ * a new one, saving the old one, which we will revert to at the end of this \r
+ * new one.\r
+ */\r
+ void (*pushCharStream) (struct ANTLR3_LEXER_struct * lexer, pANTLR3_INPUT_STREAM input);\r
+\r
+ /** Pointer to a function that abandons the current input stream, whether it\r
+ * is empty or not and reverts to the previous stacked input stream.\r
+ */\r
+ void (*popCharStream) (struct ANTLR3_LEXER_struct * lexer);\r
+\r
+ /** Pointer to a function that emits the supplied token as the next token in\r
+ * the stream.\r
+ */\r
+ void (*emitNew) (struct ANTLR3_LEXER_struct * lexer, pANTLR3_COMMON_TOKEN token);\r
+\r
+ /** Pointer to a function that constructs a new token from the lexer stored information \r
+ */\r
+ pANTLR3_COMMON_TOKEN (*emit) (struct ANTLR3_LEXER_struct * lexer);\r
+\r
+ /** Pointer to the user provided (either manually or through code generation\r
+ * function that causes the lexer rules to run the lexing rules and produce \r
+ * the next token if there iss one. This is called from nextToken() in the\r
+ * pANTLR3_TOKEN_SOURCE. Note that the input parameter for this funciton is \r
+ * the generated lexer context (stored in ctx in this interface) it is a generated\r
+ * function and expects the context to be the generated lexer. \r
+ */\r
+ void (*mTokens) (void * ctx);\r
+\r
+ /** Pointer to a function that attempts to match and consume the specified string from the input\r
+ * stream. Note that strings muse be passed as terminated arrays of ANTLR3_UCHAR. Strings are terminated\r
+ * with 0xFFFFFFFF, which is an invalid UTF32 character\r
+ */\r
+ ANTLR3_BOOLEAN (*matchs) (struct ANTLR3_LEXER_struct * lexer, ANTLR3_UCHAR * string);\r
+\r
+ /** Pointer to a function that matches and consumes the specified character from the input stream.\r
+ * As the input stream is required to provide characters via LA() as UTF32 characters it does not \r
+ * need to provide an implementation if it is not sourced from 8 bit ASCII. The default lexer\r
+ * implementation is source encoding agnostic, unless for some reason it takes two 32 bit characters\r
+ * to specify a single character, in which case the input stream and the lexer rules would have to match\r
+ * in encoding and then it would work 'by accident' anyway.\r
+ */\r
+ ANTLR3_BOOLEAN (*matchc) (struct ANTLR3_LEXER_struct * lexer, ANTLR3_UCHAR c);\r
+\r
+ /** Pointer to a function that matches any character in the supplied range (I suppose it could be a token range too\r
+ * but this would only be useful if the tokens were in tsome guaranteed order which is\r
+ * only going to happen with a hand crafted token set).\r
+ */\r
+ ANTLR3_BOOLEAN (*matchRange) (struct ANTLR3_LEXER_struct * lexer, ANTLR3_UCHAR low, ANTLR3_UCHAR high);\r
+\r
+ /** Pointer to a function that matches the next token/char in the input stream\r
+ * regardless of what it actaully is.\r
+ */\r
+ void (*matchAny) (struct ANTLR3_LEXER_struct * lexer);\r
+\r
+ /** Pointer to a function that recovers from an error found in the input stream.\r
+ * Generally, this will be a #ANTLR3_EXCEPTION_NOVIABLE_ALT but it could also\r
+ * be from a mismatched token that the (*match)() could not recover from.\r
+ */\r
+ void (*recover) (struct ANTLR3_LEXER_struct * lexer);\r
+\r
+ /** Pointer to function to return the current line number in the input stream\r
+ */\r
+ ANTLR3_UINT32 (*getLine) (struct ANTLR3_LEXER_struct * lexer);\r
+ ANTLR3_MARKER (*getCharIndex) (struct ANTLR3_LEXER_struct * lexer);\r
+ ANTLR3_UINT32 (*getCharPositionInLine)(struct ANTLR3_LEXER_struct * lexer);\r
+\r
+ /** Pointer to function to return the text so far for the current token being generated\r
+ */\r
+ pANTLR3_STRING (*getText) (struct ANTLR3_LEXER_struct * lexer);\r
+\r
+\r
+ /** Pointer to a function that knows how to free the resources of a lexer\r
+ */\r
+ void (*free) (struct ANTLR3_LEXER_struct * lexer);\r
+\r
+}\r
+ ANTLR3_LEXER;\r
+\r
+#ifdef __cplusplus\r
+}\r
+#endif\r
+\r
+#endif\r