--- /dev/null
+/**\r
+ * Contains the default implementation of the common token used within\r
+ * java. Custom tokens should create this structure and then append to it using the \r
+ * custom pointer to install their own structure and API.\r
+ */\r
+\r
+// [The "BSD licence"]\r
+// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC\r
+// http://www.temporal-wave.com\r
+// http://www.linkedin.com/in/jimidle\r
+//\r
+// All rights reserved.\r
+//\r
+// Redistribution and use in source and binary forms, with or without\r
+// modification, are permitted provided that the following conditions\r
+// are met:\r
+// 1. Redistributions of source code must retain the above copyright\r
+// notice, this list of conditions and the following disclaimer.\r
+// 2. Redistributions in binary form must reproduce the above copyright\r
+// notice, this list of conditions and the following disclaimer in the\r
+// documentation and/or other materials provided with the distribution.\r
+// 3. The name of the author may not be used to endorse or promote products\r
+// derived from this software without specific prior written permission.\r
+//\r
+// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\r
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\r
+// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\r
+// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\r
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\r
+// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\r
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\r
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\r
+// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r
+\r
+#include <antlr3.h>\r
+\r
+/* Token API\r
+ */\r
+static pANTLR3_STRING getText (pANTLR3_COMMON_TOKEN token);\r
+static void setText (pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text);\r
+static void setText8 (pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text);\r
+static ANTLR3_UINT32 getType (pANTLR3_COMMON_TOKEN token);\r
+static void setType (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type);\r
+static ANTLR3_UINT32 getLine (pANTLR3_COMMON_TOKEN token);\r
+static void setLine (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line);\r
+static ANTLR3_INT32 getCharPositionInLine (pANTLR3_COMMON_TOKEN token);\r
+static void setCharPositionInLine (pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos);\r
+static ANTLR3_UINT32 getChannel (pANTLR3_COMMON_TOKEN token);\r
+static void setChannel (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel);\r
+static ANTLR3_MARKER getTokenIndex (pANTLR3_COMMON_TOKEN token);\r
+static void setTokenIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER);\r
+static ANTLR3_MARKER getStartIndex (pANTLR3_COMMON_TOKEN token);\r
+static void setStartIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);\r
+static ANTLR3_MARKER getStopIndex (pANTLR3_COMMON_TOKEN token);\r
+static void setStopIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);\r
+static pANTLR3_STRING toString (pANTLR3_COMMON_TOKEN token);\r
+\r
+/* Factory API\r
+ */\r
+static void factoryClose (pANTLR3_TOKEN_FACTORY factory);\r
+static pANTLR3_COMMON_TOKEN newToken (void);\r
+static void setInputStream (pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input);\r
+\r
+/* Internal management functions\r
+ */\r
+static void newPool (pANTLR3_TOKEN_FACTORY factory);\r
+static pANTLR3_COMMON_TOKEN newPoolToken (pANTLR3_TOKEN_FACTORY factory);\r
+\r
+\r
+\r
+ANTLR3_API pANTLR3_COMMON_TOKEN\r
+antlr3CommonTokenNew(ANTLR3_UINT32 ttype)\r
+{\r
+ pANTLR3_COMMON_TOKEN token;\r
+\r
+ // Create a raw token with the interface installed\r
+ //\r
+ token = newToken();\r
+\r
+ if (token != NULL)\r
+ {\r
+ token->setType(token, ttype);\r
+ }\r
+\r
+ // All good\r
+ //\r
+ return token;\r
+}\r
+\r
+ANTLR3_API pANTLR3_TOKEN_FACTORY\r
+antlr3TokenFactoryNew(pANTLR3_INPUT_STREAM input)\r
+{\r
+ pANTLR3_TOKEN_FACTORY factory;\r
+\r
+ /* allocate memory\r
+ */\r
+ factory = (pANTLR3_TOKEN_FACTORY) ANTLR3_MALLOC((size_t)sizeof(ANTLR3_TOKEN_FACTORY));\r
+\r
+ if (factory == NULL)\r
+ {\r
+ return NULL;\r
+ }\r
+\r
+ /* Install factory API\r
+ */\r
+ factory->newToken = newPoolToken;\r
+ factory->close = factoryClose;\r
+ factory->setInputStream = setInputStream;\r
+ \r
+ /* Allocate the initial pool\r
+ */\r
+ factory->thisPool = -1;\r
+ factory->pools = NULL;\r
+ newPool(factory);\r
+\r
+ /* Factory space is good, we now want to initialize our cheating token\r
+ * which one it is initialized is the model for all tokens we manufacture\r
+ */\r
+ antlr3SetTokenAPI(&factory->unTruc);\r
+\r
+ /* Set some initial variables for future copying\r
+ */\r
+ factory->unTruc.factoryMade = ANTLR3_TRUE;\r
+\r
+ // Input stream\r
+ //\r
+ setInputStream(factory, input);\r
+ \r
+ return factory;\r
+\r
+}\r
+\r
+static void\r
+setInputStream (pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input)\r
+{\r
+ factory->input = input;\r
+ factory->unTruc.input = input;\r
+ if (input != NULL)\r
+ {\r
+ factory->unTruc.strFactory = input->strFactory;\r
+ }\r
+ else\r
+ {\r
+ factory->unTruc.strFactory = NULL;\r
+ }\r
+}\r
+\r
+static void\r
+newPool(pANTLR3_TOKEN_FACTORY factory)\r
+{\r
+ /* Increment factory count\r
+ */\r
+ factory->thisPool++;\r
+\r
+ /* Ensure we have enough pointers allocated\r
+ */\r
+ factory->pools = (pANTLR3_COMMON_TOKEN *)\r
+ ANTLR3_REALLOC( (void *)factory->pools, /* Current pools pointer (starts at NULL) */\r
+ (ANTLR3_UINT32)((factory->thisPool + 1) * sizeof(pANTLR3_COMMON_TOKEN *)) /* Memory for new pool pointers */\r
+ );\r
+\r
+ /* Allocate a new pool for the factory\r
+ */\r
+ factory->pools[factory->thisPool] =\r
+ (pANTLR3_COMMON_TOKEN) \r
+ ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_COMMON_TOKEN) * ANTLR3_FACTORY_POOL_SIZE));\r
+\r
+ /* Reset the counters\r
+ */\r
+ factory->nextToken = 0;\r
+ \r
+ /* Done\r
+ */\r
+ return;\r
+}\r
+\r
+static pANTLR3_COMMON_TOKEN\r
+newPoolToken(pANTLR3_TOKEN_FACTORY factory)\r
+{\r
+ pANTLR3_COMMON_TOKEN token;\r
+\r
+ /* See if we need a new token pool before allocating a new\r
+ * one\r
+ */\r
+ if (factory->nextToken >= ANTLR3_FACTORY_POOL_SIZE)\r
+ {\r
+ /* We ran out of tokens in the current pool, so we need a new pool\r
+ */\r
+ newPool(factory);\r
+ }\r
+\r
+ /* Assuming everything went well (we are trying for performance here so doing minimal\r
+ * error checking. Then we can work out what the pointer is to the next token.\r
+ */\r
+ token = factory->pools[factory->thisPool] + factory->nextToken;\r
+ factory->nextToken++;\r
+\r
+ /* We have our token pointer now, so we can initialize it to the predefined model.\r
+ */\r
+ antlr3SetTokenAPI(token);\r
+\r
+ /* It is factory made, and we need to copy the string factory pointer\r
+ */\r
+ token->factoryMade = ANTLR3_TRUE;\r
+ token->strFactory = factory->input == NULL ? NULL : factory->input->strFactory;\r
+ token->input = factory->input;\r
+\r
+ /* And we are done\r
+ */\r
+ return token;\r
+}\r
+\r
+static void\r
+factoryClose (pANTLR3_TOKEN_FACTORY factory)\r
+{\r
+ pANTLR3_COMMON_TOKEN pool;\r
+ ANTLR3_INT32 poolCount;\r
+ ANTLR3_UINT32 limit;\r
+ ANTLR3_UINT32 token;\r
+ pANTLR3_COMMON_TOKEN check;\r
+\r
+ /* We iterate the token pools one at a time\r
+ */\r
+ for (poolCount = 0; poolCount <= factory->thisPool; poolCount++)\r
+ {\r
+ /* Pointer to current pool\r
+ */\r
+ pool = factory->pools[poolCount];\r
+\r
+ /* Work out how many tokens we need to check in this pool.\r
+ */\r
+ limit = (poolCount == factory->thisPool ? factory->nextToken : ANTLR3_FACTORY_POOL_SIZE);\r
+ \r
+ /* Marginal condition, we might be at the start of a brand new pool\r
+ * where the nextToken is 0 and nothing has been allocated.\r
+ */\r
+ if (limit > 0)\r
+ {\r
+ /* We have some tokens allocated from this pool\r
+ */\r
+ for (token = 0; token < limit; token++)\r
+ {\r
+ /* Next one in the chain\r
+ */\r
+ check = pool + token;\r
+\r
+ /* If the programmer made this a custom token, then\r
+ * see if we need to call their free routine.\r
+ */\r
+ if (check->custom != NULL && check->freeCustom != NULL)\r
+ {\r
+ check->freeCustom(check->custom);\r
+ check->custom = NULL;\r
+ }\r
+ }\r
+ }\r
+\r
+ /* We can now free this pool allocation\r
+ */\r
+ ANTLR3_FREE(factory->pools[poolCount]);\r
+ factory->pools[poolCount] = NULL;\r
+ }\r
+\r
+ /* All the pools are deallocated we can free the pointers to the pools\r
+ * now.\r
+ */\r
+ ANTLR3_FREE(factory->pools);\r
+\r
+ /* Finally, we can free the space for the factory itself\r
+ */\r
+ ANTLR3_FREE(factory);\r
+}\r
+\r
+\r
+static pANTLR3_COMMON_TOKEN \r
+newToken(void)\r
+{\r
+ pANTLR3_COMMON_TOKEN token;\r
+\r
+ /* Allocate memory for this\r
+ */\r
+ token = (pANTLR3_COMMON_TOKEN) ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_COMMON_TOKEN)));\r
+\r
+ if (token == NULL)\r
+ {\r
+ return NULL;\r
+ }\r
+\r
+ // Install the API\r
+ //\r
+ antlr3SetTokenAPI(token);\r
+ token->factoryMade = ANTLR3_FALSE;\r
+\r
+ return token;\r
+}\r
+\r
+ANTLR3_API void\r
+antlr3SetTokenAPI(pANTLR3_COMMON_TOKEN token)\r
+{\r
+ token->getText = getText;\r
+ token->setText = setText;\r
+ token->setText8 = setText8;\r
+ token->getType = getType;\r
+ token->setType = setType;\r
+ token->getLine = getLine;\r
+ token->setLine = setLine;\r
+ token->setLine = setLine;\r
+ token->getCharPositionInLine = getCharPositionInLine;\r
+ token->setCharPositionInLine = setCharPositionInLine;\r
+ token->getChannel = getChannel;\r
+ token->setChannel = setChannel;\r
+ token->getTokenIndex = getTokenIndex;\r
+ token->setTokenIndex = setTokenIndex;\r
+ token->getStartIndex = getStartIndex;\r
+ token->setStartIndex = setStartIndex;\r
+ token->getStopIndex = getStopIndex;\r
+ token->setStopIndex = setStopIndex;\r
+ token->toString = toString;\r
+\r
+ // Set defaults\r
+ //\r
+ token->setCharPositionInLine(token, -1);\r
+\r
+ token->custom = NULL;\r
+ token->freeCustom = NULL;\r
+ token->type = ANTLR3_TOKEN_INVALID;\r
+ token->textState = ANTLR3_TEXT_NONE;\r
+ token->start = 0;\r
+ token->stop = 0;\r
+ token->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;\r
+ token->line = 0;\r
+ token->index = 0;\r
+ token->input = NULL;\r
+ token->user1 = 0;\r
+ token->user2 = 0;\r
+ token->user3 = 0;\r
+ token->custom = NULL;\r
+\r
+ return;\r
+}\r
+\r
+static pANTLR3_STRING getText (pANTLR3_COMMON_TOKEN token)\r
+{\r
+ switch (token->textState)\r
+ {\r
+ case ANTLR3_TEXT_STRING:\r
+\r
+ // Someone already created a string for this token, so we just\r
+ // use it.\r
+ //\r
+ return token->tokText.text;\r
+ break;\r
+ \r
+ case ANTLR3_TEXT_CHARP:\r
+\r
+ // We had a straight text pointer installed, now we\r
+ // must convert it to a string. Note we have to do this here\r
+ // or otherwise setText8() will just install the same char*\r
+ //\r
+ if (token->strFactory != NULL)\r
+ {\r
+ token->tokText.text = token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)token->tokText.chars);\r
+ token->textState = ANTLR3_TEXT_STRING;\r
+ return token->tokText.text;\r
+ }\r
+ else\r
+ {\r
+ // We cannot do anything here\r
+ //\r
+ return NULL;\r
+ }\r
+ break;\r
+\r
+ default:\r
+\r
+ // EOF is a special case\r
+ //\r
+ if (token->type == ANTLR3_TOKEN_EOF)\r
+ {\r
+ token->tokText.text = token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)"<EOF>");\r
+ token->textState = ANTLR3_TEXT_STRING;\r
+ return token->tokText.text;\r
+ }\r
+\r
+\r
+ // We had nothing installed in the token, create a new string\r
+ // from the input stream\r
+ //\r
+\r
+ if (token->input != NULL)\r
+ {\r
+ \r
+ return token->input->substr( token->input, \r
+ token->getStartIndex(token), \r
+ token->getStopIndex(token)\r
+ );\r
+ }\r
+\r
+ // Nothing to return, there is no input stream\r
+ //\r
+ return NULL;\r
+ break;\r
+ }\r
+}\r
+static void setText8 (pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text)\r
+{\r
+ // No text to set, so ignore\r
+ //\r
+ if (text == NULL) return;\r
+\r
+ switch (token->textState)\r
+ {\r
+ case ANTLR3_TEXT_NONE:\r
+ case ANTLR3_TEXT_CHARP: // Caller must free before setting again, if it needs to be freed\r
+\r
+ // Nothing in there yet, or just a char *, so just set the\r
+ // text as a pointer\r
+ //\r
+ token->textState = ANTLR3_TEXT_CHARP;\r
+ token->tokText.chars = (pANTLR3_UCHAR)text;\r
+ break;\r
+\r
+ default:\r
+\r
+ // It was already a pANTLR3_STRING, so just override it\r
+ //\r
+ token->tokText.text->set8(token->tokText.text, (const char *)text);\r
+ break;\r
+ }\r
+\r
+ // We are done \r
+ //\r
+ return;\r
+}\r
+\r
+/** \brief Install the supplied text string as teh text for the token.\r
+ * The method assumes that the existing text (if any) was created by a factory\r
+ * and so does not attempt to release any memory it is using.Text not created\r
+ * by a string fctory (not advised) should be released prior to this call.\r
+ */\r
+static void setText (pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text)\r
+{\r
+ // Merely replaces and existing pre-defined text with the supplied\r
+ // string\r
+ //\r
+ token->textState = ANTLR3_TEXT_STRING;\r
+ token->tokText.text = text;\r
+\r
+ /* We are done \r
+ */\r
+ return;\r
+}\r
+\r
+static ANTLR3_UINT32 getType (pANTLR3_COMMON_TOKEN token)\r
+{\r
+ return token->type;\r
+}\r
+\r
+static void setType (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type)\r
+{\r
+ token->type = type;\r
+}\r
+\r
+static ANTLR3_UINT32 getLine (pANTLR3_COMMON_TOKEN token)\r
+{\r
+ return token->line;\r
+}\r
+\r
+static void setLine (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line)\r
+{\r
+ token->line = line;\r
+}\r
+\r
+static ANTLR3_INT32 getCharPositionInLine (pANTLR3_COMMON_TOKEN token)\r
+{\r
+ return token->charPosition;\r
+}\r
+\r
+static void setCharPositionInLine (pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos)\r
+{\r
+ token->charPosition = pos;\r
+}\r
+\r
+static ANTLR3_UINT32 getChannel (pANTLR3_COMMON_TOKEN token)\r
+{\r
+ return token->channel;\r
+}\r
+\r
+static void setChannel (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel)\r
+{\r
+ token->channel = channel;\r
+}\r
+\r
+static ANTLR3_MARKER getTokenIndex (pANTLR3_COMMON_TOKEN token)\r
+{\r
+ return token->index;\r
+}\r
+\r
+static void setTokenIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index)\r
+{\r
+ token->index = index;\r
+}\r
+\r
+static ANTLR3_MARKER getStartIndex (pANTLR3_COMMON_TOKEN token)\r
+{\r
+ return token->start == -1 ? (ANTLR3_MARKER)(token->input->data) : token->start;\r
+}\r
+\r
+static void setStartIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER start)\r
+{\r
+ token->start = start;\r
+}\r
+\r
+static ANTLR3_MARKER getStopIndex (pANTLR3_COMMON_TOKEN token)\r
+{\r
+ return token->stop;\r
+}\r
+\r
+static void setStopIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER stop)\r
+{\r
+ token->stop = stop;\r
+}\r
+\r
+static pANTLR3_STRING toString (pANTLR3_COMMON_TOKEN token)\r
+{\r
+ pANTLR3_STRING text;\r
+ pANTLR3_STRING outtext;\r
+\r
+ text = token->getText(token);\r
+ \r
+ if (text == NULL)\r
+ {\r
+ return NULL;\r
+ }\r
+\r
+ if (text->factory == NULL)\r
+ {\r
+ return text; // This usall ymeans it is the EOF token\r
+ }\r
+\r
+ /* A new empty string to assemble all the stuff in\r
+ */\r
+ outtext = text->factory->newRaw(text->factory);\r
+\r
+ /* Now we use our handy dandy string utility to assemble the\r
+ * the reporting string\r
+ * return "[@"+getTokenIndex()+","+start+":"+stop+"='"+txt+"',<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+"]";\r
+ */\r
+ outtext->append8(outtext, "[Index: ");\r
+ outtext->addi (outtext, (ANTLR3_INT32)token->getTokenIndex(token));\r
+ outtext->append8(outtext, " (Start: ");\r
+ outtext->addi (outtext, (ANTLR3_INT32)token->getStartIndex(token));\r
+ outtext->append8(outtext, "-Stop: ");\r
+ outtext->addi (outtext, (ANTLR3_INT32)token->getStopIndex(token));\r
+ outtext->append8(outtext, ") ='");\r
+ outtext->appendS(outtext, text);\r
+ outtext->append8(outtext, "', type<");\r
+ outtext->addi (outtext, token->type);\r
+ outtext->append8(outtext, "> ");\r
+\r
+ if (token->getChannel(token) > ANTLR3_TOKEN_DEFAULT_CHANNEL)\r
+ {\r
+ outtext->append8(outtext, "(channel = ");\r
+ outtext->addi (outtext, (ANTLR3_INT32)token->getChannel(token));\r
+ outtext->append8(outtext, ") ");\r
+ }\r
+\r
+ outtext->append8(outtext, "Line: ");\r
+ outtext->addi (outtext, (ANTLR3_INT32)token->getLine(token));\r
+ outtext->append8(outtext, " LinePos:");\r
+ outtext->addi (outtext, token->getCharPositionInLine(token));\r
+ outtext->addc (outtext, ']');\r
+\r
+ return outtext;\r
+}\r
+\r
Code Review / simantics / platform.git / blobdiff