-/**\r
- * Contains the default implementation of the common token used within\r
- * java. Custom tokens should create this structure and then append to it using the \r
- * custom pointer to install their own structure and API.\r
- */\r
-\r
-// [The "BSD licence"]\r
-// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC\r
-// http://www.temporal-wave.com\r
-// http://www.linkedin.com/in/jimidle\r
-//\r
-// All rights reserved.\r
-//\r
-// Redistribution and use in source and binary forms, with or without\r
-// modification, are permitted provided that the following conditions\r
-// are met:\r
-// 1. Redistributions of source code must retain the above copyright\r
-// notice, this list of conditions and the following disclaimer.\r
-// 2. Redistributions in binary form must reproduce the above copyright\r
-// notice, this list of conditions and the following disclaimer in the\r
-// documentation and/or other materials provided with the distribution.\r
-// 3. The name of the author may not be used to endorse or promote products\r
-// derived from this software without specific prior written permission.\r
-//\r
-// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\r
-// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\r
-// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\r
-// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\r
-// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\r
-// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\r
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\r
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\r
-// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r
-\r
-#include <antlr3.h>\r
-\r
-/* Token API\r
- */\r
-static pANTLR3_STRING getText (pANTLR3_COMMON_TOKEN token);\r
-static void setText (pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text);\r
-static void setText8 (pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text);\r
-static ANTLR3_UINT32 getType (pANTLR3_COMMON_TOKEN token);\r
-static void setType (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type);\r
-static ANTLR3_UINT32 getLine (pANTLR3_COMMON_TOKEN token);\r
-static void setLine (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line);\r
-static ANTLR3_INT32 getCharPositionInLine (pANTLR3_COMMON_TOKEN token);\r
-static void setCharPositionInLine (pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos);\r
-static ANTLR3_UINT32 getChannel (pANTLR3_COMMON_TOKEN token);\r
-static void setChannel (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel);\r
-static ANTLR3_MARKER getTokenIndex (pANTLR3_COMMON_TOKEN token);\r
-static void setTokenIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER);\r
-static ANTLR3_MARKER getStartIndex (pANTLR3_COMMON_TOKEN token);\r
-static void setStartIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);\r
-static ANTLR3_MARKER getStopIndex (pANTLR3_COMMON_TOKEN token);\r
-static void setStopIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);\r
-static pANTLR3_STRING toString (pANTLR3_COMMON_TOKEN token);\r
-\r
-/* Factory API\r
- */\r
-static void factoryClose (pANTLR3_TOKEN_FACTORY factory);\r
-static pANTLR3_COMMON_TOKEN newToken (void);\r
-static void setInputStream (pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input);\r
-\r
-/* Internal management functions\r
- */\r
-static void newPool (pANTLR3_TOKEN_FACTORY factory);\r
-static pANTLR3_COMMON_TOKEN newPoolToken (pANTLR3_TOKEN_FACTORY factory);\r
-\r
-\r
-\r
-ANTLR3_API pANTLR3_COMMON_TOKEN\r
-antlr3CommonTokenNew(ANTLR3_UINT32 ttype)\r
-{\r
- pANTLR3_COMMON_TOKEN token;\r
-\r
- // Create a raw token with the interface installed\r
- //\r
- token = newToken();\r
-\r
- if (token != NULL)\r
- {\r
- token->setType(token, ttype);\r
- }\r
-\r
- // All good\r
- //\r
- return token;\r
-}\r
-\r
-ANTLR3_API pANTLR3_TOKEN_FACTORY\r
-antlr3TokenFactoryNew(pANTLR3_INPUT_STREAM input)\r
-{\r
- pANTLR3_TOKEN_FACTORY factory;\r
-\r
- /* allocate memory\r
- */\r
- factory = (pANTLR3_TOKEN_FACTORY) ANTLR3_MALLOC((size_t)sizeof(ANTLR3_TOKEN_FACTORY));\r
-\r
- if (factory == NULL)\r
- {\r
- return NULL;\r
- }\r
-\r
- /* Install factory API\r
- */\r
- factory->newToken = newPoolToken;\r
- factory->close = factoryClose;\r
- factory->setInputStream = setInputStream;\r
- \r
- /* Allocate the initial pool\r
- */\r
- factory->thisPool = -1;\r
- factory->pools = NULL;\r
- newPool(factory);\r
-\r
- /* Factory space is good, we now want to initialize our cheating token\r
- * which one it is initialized is the model for all tokens we manufacture\r
- */\r
- antlr3SetTokenAPI(&factory->unTruc);\r
-\r
- /* Set some initial variables for future copying\r
- */\r
- factory->unTruc.factoryMade = ANTLR3_TRUE;\r
-\r
- // Input stream\r
- //\r
- setInputStream(factory, input);\r
- \r
- return factory;\r
-\r
-}\r
-\r
-static void\r
-setInputStream (pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input)\r
-{\r
- factory->input = input;\r
- factory->unTruc.input = input;\r
- if (input != NULL)\r
- {\r
- factory->unTruc.strFactory = input->strFactory;\r
- }\r
- else\r
- {\r
- factory->unTruc.strFactory = NULL;\r
- }\r
-}\r
-\r
-static void\r
-newPool(pANTLR3_TOKEN_FACTORY factory)\r
-{\r
- /* Increment factory count\r
- */\r
- factory->thisPool++;\r
-\r
- /* Ensure we have enough pointers allocated\r
- */\r
- factory->pools = (pANTLR3_COMMON_TOKEN *)\r
- ANTLR3_REALLOC( (void *)factory->pools, /* Current pools pointer (starts at NULL) */\r
- (ANTLR3_UINT32)((factory->thisPool + 1) * sizeof(pANTLR3_COMMON_TOKEN *)) /* Memory for new pool pointers */\r
- );\r
-\r
- /* Allocate a new pool for the factory\r
- */\r
- factory->pools[factory->thisPool] =\r
- (pANTLR3_COMMON_TOKEN) \r
- ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_COMMON_TOKEN) * ANTLR3_FACTORY_POOL_SIZE));\r
-\r
- /* Reset the counters\r
- */\r
- factory->nextToken = 0;\r
- \r
- /* Done\r
- */\r
- return;\r
-}\r
-\r
-static pANTLR3_COMMON_TOKEN\r
-newPoolToken(pANTLR3_TOKEN_FACTORY factory)\r
-{\r
- pANTLR3_COMMON_TOKEN token;\r
-\r
- /* See if we need a new token pool before allocating a new\r
- * one\r
- */\r
- if (factory->nextToken >= ANTLR3_FACTORY_POOL_SIZE)\r
- {\r
- /* We ran out of tokens in the current pool, so we need a new pool\r
- */\r
- newPool(factory);\r
- }\r
-\r
- /* Assuming everything went well (we are trying for performance here so doing minimal\r
- * error checking. Then we can work out what the pointer is to the next token.\r
- */\r
- token = factory->pools[factory->thisPool] + factory->nextToken;\r
- factory->nextToken++;\r
-\r
- /* We have our token pointer now, so we can initialize it to the predefined model.\r
- */\r
- antlr3SetTokenAPI(token);\r
-\r
- /* It is factory made, and we need to copy the string factory pointer\r
- */\r
- token->factoryMade = ANTLR3_TRUE;\r
- token->strFactory = factory->input == NULL ? NULL : factory->input->strFactory;\r
- token->input = factory->input;\r
-\r
- /* And we are done\r
- */\r
- return token;\r
-}\r
-\r
-static void\r
-factoryClose (pANTLR3_TOKEN_FACTORY factory)\r
-{\r
- pANTLR3_COMMON_TOKEN pool;\r
- ANTLR3_INT32 poolCount;\r
- ANTLR3_UINT32 limit;\r
- ANTLR3_UINT32 token;\r
- pANTLR3_COMMON_TOKEN check;\r
-\r
- /* We iterate the token pools one at a time\r
- */\r
- for (poolCount = 0; poolCount <= factory->thisPool; poolCount++)\r
- {\r
- /* Pointer to current pool\r
- */\r
- pool = factory->pools[poolCount];\r
-\r
- /* Work out how many tokens we need to check in this pool.\r
- */\r
- limit = (poolCount == factory->thisPool ? factory->nextToken : ANTLR3_FACTORY_POOL_SIZE);\r
- \r
- /* Marginal condition, we might be at the start of a brand new pool\r
- * where the nextToken is 0 and nothing has been allocated.\r
- */\r
- if (limit > 0)\r
- {\r
- /* We have some tokens allocated from this pool\r
- */\r
- for (token = 0; token < limit; token++)\r
- {\r
- /* Next one in the chain\r
- */\r
- check = pool + token;\r
-\r
- /* If the programmer made this a custom token, then\r
- * see if we need to call their free routine.\r
- */\r
- if (check->custom != NULL && check->freeCustom != NULL)\r
- {\r
- check->freeCustom(check->custom);\r
- check->custom = NULL;\r
- }\r
- }\r
- }\r
-\r
- /* We can now free this pool allocation\r
- */\r
- ANTLR3_FREE(factory->pools[poolCount]);\r
- factory->pools[poolCount] = NULL;\r
- }\r
-\r
- /* All the pools are deallocated we can free the pointers to the pools\r
- * now.\r
- */\r
- ANTLR3_FREE(factory->pools);\r
-\r
- /* Finally, we can free the space for the factory itself\r
- */\r
- ANTLR3_FREE(factory);\r
-}\r
-\r
-\r
-static pANTLR3_COMMON_TOKEN \r
-newToken(void)\r
-{\r
- pANTLR3_COMMON_TOKEN token;\r
-\r
- /* Allocate memory for this\r
- */\r
- token = (pANTLR3_COMMON_TOKEN) ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_COMMON_TOKEN)));\r
-\r
- if (token == NULL)\r
- {\r
- return NULL;\r
- }\r
-\r
- // Install the API\r
- //\r
- antlr3SetTokenAPI(token);\r
- token->factoryMade = ANTLR3_FALSE;\r
-\r
- return token;\r
-}\r
-\r
-ANTLR3_API void\r
-antlr3SetTokenAPI(pANTLR3_COMMON_TOKEN token)\r
-{\r
- token->getText = getText;\r
- token->setText = setText;\r
- token->setText8 = setText8;\r
- token->getType = getType;\r
- token->setType = setType;\r
- token->getLine = getLine;\r
- token->setLine = setLine;\r
- token->setLine = setLine;\r
- token->getCharPositionInLine = getCharPositionInLine;\r
- token->setCharPositionInLine = setCharPositionInLine;\r
- token->getChannel = getChannel;\r
- token->setChannel = setChannel;\r
- token->getTokenIndex = getTokenIndex;\r
- token->setTokenIndex = setTokenIndex;\r
- token->getStartIndex = getStartIndex;\r
- token->setStartIndex = setStartIndex;\r
- token->getStopIndex = getStopIndex;\r
- token->setStopIndex = setStopIndex;\r
- token->toString = toString;\r
-\r
- // Set defaults\r
- //\r
- token->setCharPositionInLine(token, -1);\r
-\r
- token->custom = NULL;\r
- token->freeCustom = NULL;\r
- token->type = ANTLR3_TOKEN_INVALID;\r
- token->textState = ANTLR3_TEXT_NONE;\r
- token->start = 0;\r
- token->stop = 0;\r
- token->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;\r
- token->line = 0;\r
- token->index = 0;\r
- token->input = NULL;\r
- token->user1 = 0;\r
- token->user2 = 0;\r
- token->user3 = 0;\r
- token->custom = NULL;\r
-\r
- return;\r
-}\r
-\r
-static pANTLR3_STRING getText (pANTLR3_COMMON_TOKEN token)\r
-{\r
- switch (token->textState)\r
- {\r
- case ANTLR3_TEXT_STRING:\r
-\r
- // Someone already created a string for this token, so we just\r
- // use it.\r
- //\r
- return token->tokText.text;\r
- break;\r
- \r
- case ANTLR3_TEXT_CHARP:\r
-\r
- // We had a straight text pointer installed, now we\r
- // must convert it to a string. Note we have to do this here\r
- // or otherwise setText8() will just install the same char*\r
- //\r
- if (token->strFactory != NULL)\r
- {\r
- token->tokText.text = token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)token->tokText.chars);\r
- token->textState = ANTLR3_TEXT_STRING;\r
- return token->tokText.text;\r
- }\r
- else\r
- {\r
- // We cannot do anything here\r
- //\r
- return NULL;\r
- }\r
- break;\r
-\r
- default:\r
-\r
- // EOF is a special case\r
- //\r
- if (token->type == ANTLR3_TOKEN_EOF)\r
- {\r
- token->tokText.text = token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)"<EOF>");\r
- token->textState = ANTLR3_TEXT_STRING;\r
- return token->tokText.text;\r
- }\r
-\r
-\r
- // We had nothing installed in the token, create a new string\r
- // from the input stream\r
- //\r
-\r
- if (token->input != NULL)\r
- {\r
- \r
- return token->input->substr( token->input, \r
- token->getStartIndex(token), \r
- token->getStopIndex(token)\r
- );\r
- }\r
-\r
- // Nothing to return, there is no input stream\r
- //\r
- return NULL;\r
- break;\r
- }\r
-}\r
-static void setText8 (pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text)\r
-{\r
- // No text to set, so ignore\r
- //\r
- if (text == NULL) return;\r
-\r
- switch (token->textState)\r
- {\r
- case ANTLR3_TEXT_NONE:\r
- case ANTLR3_TEXT_CHARP: // Caller must free before setting again, if it needs to be freed\r
-\r
- // Nothing in there yet, or just a char *, so just set the\r
- // text as a pointer\r
- //\r
- token->textState = ANTLR3_TEXT_CHARP;\r
- token->tokText.chars = (pANTLR3_UCHAR)text;\r
- break;\r
-\r
- default:\r
-\r
- // It was already a pANTLR3_STRING, so just override it\r
- //\r
- token->tokText.text->set8(token->tokText.text, (const char *)text);\r
- break;\r
- }\r
-\r
- // We are done \r
- //\r
- return;\r
-}\r
-\r
-/** \brief Install the supplied text string as teh text for the token.\r
- * The method assumes that the existing text (if any) was created by a factory\r
- * and so does not attempt to release any memory it is using.Text not created\r
- * by a string fctory (not advised) should be released prior to this call.\r
- */\r
-static void setText (pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text)\r
-{\r
- // Merely replaces and existing pre-defined text with the supplied\r
- // string\r
- //\r
- token->textState = ANTLR3_TEXT_STRING;\r
- token->tokText.text = text;\r
-\r
- /* We are done \r
- */\r
- return;\r
-}\r
-\r
-static ANTLR3_UINT32 getType (pANTLR3_COMMON_TOKEN token)\r
-{\r
- return token->type;\r
-}\r
-\r
-static void setType (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type)\r
-{\r
- token->type = type;\r
-}\r
-\r
-static ANTLR3_UINT32 getLine (pANTLR3_COMMON_TOKEN token)\r
-{\r
- return token->line;\r
-}\r
-\r
-static void setLine (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line)\r
-{\r
- token->line = line;\r
-}\r
-\r
-static ANTLR3_INT32 getCharPositionInLine (pANTLR3_COMMON_TOKEN token)\r
-{\r
- return token->charPosition;\r
-}\r
-\r
-static void setCharPositionInLine (pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos)\r
-{\r
- token->charPosition = pos;\r
-}\r
-\r
-static ANTLR3_UINT32 getChannel (pANTLR3_COMMON_TOKEN token)\r
-{\r
- return token->channel;\r
-}\r
-\r
-static void setChannel (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel)\r
-{\r
- token->channel = channel;\r
-}\r
-\r
-static ANTLR3_MARKER getTokenIndex (pANTLR3_COMMON_TOKEN token)\r
-{\r
- return token->index;\r
-}\r
-\r
-static void setTokenIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index)\r
-{\r
- token->index = index;\r
-}\r
-\r
-static ANTLR3_MARKER getStartIndex (pANTLR3_COMMON_TOKEN token)\r
-{\r
- return token->start == -1 ? (ANTLR3_MARKER)(token->input->data) : token->start;\r
-}\r
-\r
-static void setStartIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER start)\r
-{\r
- token->start = start;\r
-}\r
-\r
-static ANTLR3_MARKER getStopIndex (pANTLR3_COMMON_TOKEN token)\r
-{\r
- return token->stop;\r
-}\r
-\r
-static void setStopIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER stop)\r
-{\r
- token->stop = stop;\r
-}\r
-\r
-static pANTLR3_STRING toString (pANTLR3_COMMON_TOKEN token)\r
-{\r
- pANTLR3_STRING text;\r
- pANTLR3_STRING outtext;\r
-\r
- text = token->getText(token);\r
- \r
- if (text == NULL)\r
- {\r
- return NULL;\r
- }\r
-\r
- if (text->factory == NULL)\r
- {\r
- return text; // This usall ymeans it is the EOF token\r
- }\r
-\r
- /* A new empty string to assemble all the stuff in\r
- */\r
- outtext = text->factory->newRaw(text->factory);\r
-\r
- /* Now we use our handy dandy string utility to assemble the\r
- * the reporting string\r
- * return "[@"+getTokenIndex()+","+start+":"+stop+"='"+txt+"',<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+"]";\r
- */\r
- outtext->append8(outtext, "[Index: ");\r
- outtext->addi (outtext, (ANTLR3_INT32)token->getTokenIndex(token));\r
- outtext->append8(outtext, " (Start: ");\r
- outtext->addi (outtext, (ANTLR3_INT32)token->getStartIndex(token));\r
- outtext->append8(outtext, "-Stop: ");\r
- outtext->addi (outtext, (ANTLR3_INT32)token->getStopIndex(token));\r
- outtext->append8(outtext, ") ='");\r
- outtext->appendS(outtext, text);\r
- outtext->append8(outtext, "', type<");\r
- outtext->addi (outtext, token->type);\r
- outtext->append8(outtext, "> ");\r
-\r
- if (token->getChannel(token) > ANTLR3_TOKEN_DEFAULT_CHANNEL)\r
- {\r
- outtext->append8(outtext, "(channel = ");\r
- outtext->addi (outtext, (ANTLR3_INT32)token->getChannel(token));\r
- outtext->append8(outtext, ") ");\r
- }\r
-\r
- outtext->append8(outtext, "Line: ");\r
- outtext->addi (outtext, (ANTLR3_INT32)token->getLine(token));\r
- outtext->append8(outtext, " LinePos:");\r
- outtext->addi (outtext, token->getCharPositionInLine(token));\r
- outtext->addc (outtext, ']');\r
-\r
- return outtext;\r
-}\r
-\r
+/**
+ * Contains the default implementation of the common token used within
+ * java. Custom tokens should create this structure and then append to it using the
+ * custom pointer to install their own structure and API.
+ */
+
+// [The "BSD licence"]
+// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
+// http://www.temporal-wave.com
+// http://www.linkedin.com/in/jimidle
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. The name of the author may not be used to endorse or promote products
+// derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <antlr3.h>
+
+/* Token API
+ */
+static pANTLR3_STRING getText (pANTLR3_COMMON_TOKEN token);
+static void setText (pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text);
+static void setText8 (pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text);
+static ANTLR3_UINT32 getType (pANTLR3_COMMON_TOKEN token);
+static void setType (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type);
+static ANTLR3_UINT32 getLine (pANTLR3_COMMON_TOKEN token);
+static void setLine (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line);
+static ANTLR3_INT32 getCharPositionInLine (pANTLR3_COMMON_TOKEN token);
+static void setCharPositionInLine (pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos);
+static ANTLR3_UINT32 getChannel (pANTLR3_COMMON_TOKEN token);
+static void setChannel (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel);
+static ANTLR3_MARKER getTokenIndex (pANTLR3_COMMON_TOKEN token);
+static void setTokenIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER);
+static ANTLR3_MARKER getStartIndex (pANTLR3_COMMON_TOKEN token);
+static void setStartIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);
+static ANTLR3_MARKER getStopIndex (pANTLR3_COMMON_TOKEN token);
+static void setStopIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);
+static pANTLR3_STRING toString (pANTLR3_COMMON_TOKEN token);
+
+/* Factory API
+ */
+static void factoryClose (pANTLR3_TOKEN_FACTORY factory);
+static pANTLR3_COMMON_TOKEN newToken (void);
+static void setInputStream (pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input);
+
+/* Internal management functions
+ */
+static void newPool (pANTLR3_TOKEN_FACTORY factory);
+static pANTLR3_COMMON_TOKEN newPoolToken (pANTLR3_TOKEN_FACTORY factory);
+
+
+
+ANTLR3_API pANTLR3_COMMON_TOKEN
+antlr3CommonTokenNew(ANTLR3_UINT32 ttype)
+{
+ pANTLR3_COMMON_TOKEN token;
+
+ // Create a raw token with the interface installed
+ //
+ token = newToken();
+
+ if (token != NULL)
+ {
+ token->setType(token, ttype);
+ }
+
+ // All good
+ //
+ return token;
+}
+
+ANTLR3_API pANTLR3_TOKEN_FACTORY
+antlr3TokenFactoryNew(pANTLR3_INPUT_STREAM input)
+{
+ pANTLR3_TOKEN_FACTORY factory;
+
+ /* allocate memory
+ */
+ factory = (pANTLR3_TOKEN_FACTORY) ANTLR3_MALLOC((size_t)sizeof(ANTLR3_TOKEN_FACTORY));
+
+ if (factory == NULL)
+ {
+ return NULL;
+ }
+
+ /* Install factory API
+ */
+ factory->newToken = newPoolToken;
+ factory->close = factoryClose;
+ factory->setInputStream = setInputStream;
+
+ /* Allocate the initial pool
+ */
+ factory->thisPool = -1;
+ factory->pools = NULL;
+ newPool(factory);
+
+ /* Factory space is good, we now want to initialize our cheating token
+ * which one it is initialized is the model for all tokens we manufacture
+ */
+ antlr3SetTokenAPI(&factory->unTruc);
+
+ /* Set some initial variables for future copying
+ */
+ factory->unTruc.factoryMade = ANTLR3_TRUE;
+
+ // Input stream
+ //
+ setInputStream(factory, input);
+
+ return factory;
+
+}
+
+static void
+setInputStream (pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input)
+{
+ factory->input = input;
+ factory->unTruc.input = input;
+ if (input != NULL)
+ {
+ factory->unTruc.strFactory = input->strFactory;
+ }
+ else
+ {
+ factory->unTruc.strFactory = NULL;
+ }
+}
+
+static void
+newPool(pANTLR3_TOKEN_FACTORY factory)
+{
+ /* Increment factory count
+ */
+ factory->thisPool++;
+
+ /* Ensure we have enough pointers allocated
+ */
+ factory->pools = (pANTLR3_COMMON_TOKEN *)
+ ANTLR3_REALLOC( (void *)factory->pools, /* Current pools pointer (starts at NULL) */
+ (ANTLR3_UINT32)((factory->thisPool + 1) * sizeof(pANTLR3_COMMON_TOKEN *)) /* Memory for new pool pointers */
+ );
+
+ /* Allocate a new pool for the factory
+ */
+ factory->pools[factory->thisPool] =
+ (pANTLR3_COMMON_TOKEN)
+ ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_COMMON_TOKEN) * ANTLR3_FACTORY_POOL_SIZE));
+
+ /* Reset the counters
+ */
+ factory->nextToken = 0;
+
+ /* Done
+ */
+ return;
+}
+
+static pANTLR3_COMMON_TOKEN
+newPoolToken(pANTLR3_TOKEN_FACTORY factory)
+{
+ pANTLR3_COMMON_TOKEN token;
+
+ /* See if we need a new token pool before allocating a new
+ * one
+ */
+ if (factory->nextToken >= ANTLR3_FACTORY_POOL_SIZE)
+ {
+ /* We ran out of tokens in the current pool, so we need a new pool
+ */
+ newPool(factory);
+ }
+
+ /* Assuming everything went well (we are trying for performance here so doing minimal
+ * error checking. Then we can work out what the pointer is to the next token.
+ */
+ token = factory->pools[factory->thisPool] + factory->nextToken;
+ factory->nextToken++;
+
+ /* We have our token pointer now, so we can initialize it to the predefined model.
+ */
+ antlr3SetTokenAPI(token);
+
+ /* It is factory made, and we need to copy the string factory pointer
+ */
+ token->factoryMade = ANTLR3_TRUE;
+ token->strFactory = factory->input == NULL ? NULL : factory->input->strFactory;
+ token->input = factory->input;
+
+ /* And we are done
+ */
+ return token;
+}
+
+static void
+factoryClose (pANTLR3_TOKEN_FACTORY factory)
+{
+ pANTLR3_COMMON_TOKEN pool;
+ ANTLR3_INT32 poolCount;
+ ANTLR3_UINT32 limit;
+ ANTLR3_UINT32 token;
+ pANTLR3_COMMON_TOKEN check;
+
+ /* We iterate the token pools one at a time
+ */
+ for (poolCount = 0; poolCount <= factory->thisPool; poolCount++)
+ {
+ /* Pointer to current pool
+ */
+ pool = factory->pools[poolCount];
+
+ /* Work out how many tokens we need to check in this pool.
+ */
+ limit = (poolCount == factory->thisPool ? factory->nextToken : ANTLR3_FACTORY_POOL_SIZE);
+
+ /* Marginal condition, we might be at the start of a brand new pool
+ * where the nextToken is 0 and nothing has been allocated.
+ */
+ if (limit > 0)
+ {
+ /* We have some tokens allocated from this pool
+ */
+ for (token = 0; token < limit; token++)
+ {
+ /* Next one in the chain
+ */
+ check = pool + token;
+
+ /* If the programmer made this a custom token, then
+ * see if we need to call their free routine.
+ */
+ if (check->custom != NULL && check->freeCustom != NULL)
+ {
+ check->freeCustom(check->custom);
+ check->custom = NULL;
+ }
+ }
+ }
+
+ /* We can now free this pool allocation
+ */
+ ANTLR3_FREE(factory->pools[poolCount]);
+ factory->pools[poolCount] = NULL;
+ }
+
+ /* All the pools are deallocated we can free the pointers to the pools
+ * now.
+ */
+ ANTLR3_FREE(factory->pools);
+
+ /* Finally, we can free the space for the factory itself
+ */
+ ANTLR3_FREE(factory);
+}
+
+
+static pANTLR3_COMMON_TOKEN
+newToken(void)
+{
+ pANTLR3_COMMON_TOKEN token;
+
+ /* Allocate memory for this
+ */
+ token = (pANTLR3_COMMON_TOKEN) ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_COMMON_TOKEN)));
+
+ if (token == NULL)
+ {
+ return NULL;
+ }
+
+ // Install the API
+ //
+ antlr3SetTokenAPI(token);
+ token->factoryMade = ANTLR3_FALSE;
+
+ return token;
+}
+
+ANTLR3_API void
+antlr3SetTokenAPI(pANTLR3_COMMON_TOKEN token)
+{
+ token->getText = getText;
+ token->setText = setText;
+ token->setText8 = setText8;
+ token->getType = getType;
+ token->setType = setType;
+ token->getLine = getLine;
+ token->setLine = setLine;
+ token->setLine = setLine;
+ token->getCharPositionInLine = getCharPositionInLine;
+ token->setCharPositionInLine = setCharPositionInLine;
+ token->getChannel = getChannel;
+ token->setChannel = setChannel;
+ token->getTokenIndex = getTokenIndex;
+ token->setTokenIndex = setTokenIndex;
+ token->getStartIndex = getStartIndex;
+ token->setStartIndex = setStartIndex;
+ token->getStopIndex = getStopIndex;
+ token->setStopIndex = setStopIndex;
+ token->toString = toString;
+
+ // Set defaults
+ //
+ token->setCharPositionInLine(token, -1);
+
+ token->custom = NULL;
+ token->freeCustom = NULL;
+ token->type = ANTLR3_TOKEN_INVALID;
+ token->textState = ANTLR3_TEXT_NONE;
+ token->start = 0;
+ token->stop = 0;
+ token->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;
+ token->line = 0;
+ token->index = 0;
+ token->input = NULL;
+ token->user1 = 0;
+ token->user2 = 0;
+ token->user3 = 0;
+ token->custom = NULL;
+
+ return;
+}
+
+static pANTLR3_STRING getText (pANTLR3_COMMON_TOKEN token)
+{
+ switch (token->textState)
+ {
+ case ANTLR3_TEXT_STRING:
+
+ // Someone already created a string for this token, so we just
+ // use it.
+ //
+ return token->tokText.text;
+ break;
+
+ case ANTLR3_TEXT_CHARP:
+
+ // We had a straight text pointer installed, now we
+ // must convert it to a string. Note we have to do this here
+ // or otherwise setText8() will just install the same char*
+ //
+ if (token->strFactory != NULL)
+ {
+ token->tokText.text = token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)token->tokText.chars);
+ token->textState = ANTLR3_TEXT_STRING;
+ return token->tokText.text;
+ }
+ else
+ {
+ // We cannot do anything here
+ //
+ return NULL;
+ }
+ break;
+
+ default:
+
+ // EOF is a special case
+ //
+ if (token->type == ANTLR3_TOKEN_EOF)
+ {
+ token->tokText.text = token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)"<EOF>");
+ token->textState = ANTLR3_TEXT_STRING;
+ return token->tokText.text;
+ }
+
+
+ // We had nothing installed in the token, create a new string
+ // from the input stream
+ //
+
+ if (token->input != NULL)
+ {
+
+ return token->input->substr( token->input,
+ token->getStartIndex(token),
+ token->getStopIndex(token)
+ );
+ }
+
+ // Nothing to return, there is no input stream
+ //
+ return NULL;
+ break;
+ }
+}
+static void setText8 (pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text)
+{
+ // No text to set, so ignore
+ //
+ if (text == NULL) return;
+
+ switch (token->textState)
+ {
+ case ANTLR3_TEXT_NONE:
+ case ANTLR3_TEXT_CHARP: // Caller must free before setting again, if it needs to be freed
+
+ // Nothing in there yet, or just a char *, so just set the
+ // text as a pointer
+ //
+ token->textState = ANTLR3_TEXT_CHARP;
+ token->tokText.chars = (pANTLR3_UCHAR)text;
+ break;
+
+ default:
+
+ // It was already a pANTLR3_STRING, so just override it
+ //
+ token->tokText.text->set8(token->tokText.text, (const char *)text);
+ break;
+ }
+
+ // We are done
+ //
+ return;
+}
+
+/** \brief Install the supplied text string as teh text for the token.
+ * The method assumes that the existing text (if any) was created by a factory
+ * and so does not attempt to release any memory it is using.Text not created
+ * by a string fctory (not advised) should be released prior to this call.
+ */
+static void setText (pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text)
+{
+ // Merely replaces and existing pre-defined text with the supplied
+ // string
+ //
+ token->textState = ANTLR3_TEXT_STRING;
+ token->tokText.text = text;
+
+ /* We are done
+ */
+ return;
+}
+
+static ANTLR3_UINT32 getType (pANTLR3_COMMON_TOKEN token)
+{
+ return token->type;
+}
+
+static void setType (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type)
+{
+ token->type = type;
+}
+
+static ANTLR3_UINT32 getLine (pANTLR3_COMMON_TOKEN token)
+{
+ return token->line;
+}
+
+static void setLine (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line)
+{
+ token->line = line;
+}
+
+static ANTLR3_INT32 getCharPositionInLine (pANTLR3_COMMON_TOKEN token)
+{
+ return token->charPosition;
+}
+
+static void setCharPositionInLine (pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos)
+{
+ token->charPosition = pos;
+}
+
+static ANTLR3_UINT32 getChannel (pANTLR3_COMMON_TOKEN token)
+{
+ return token->channel;
+}
+
+static void setChannel (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel)
+{
+ token->channel = channel;
+}
+
+static ANTLR3_MARKER getTokenIndex (pANTLR3_COMMON_TOKEN token)
+{
+ return token->index;
+}
+
+static void setTokenIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index)
+{
+ token->index = index;
+}
+
+static ANTLR3_MARKER getStartIndex (pANTLR3_COMMON_TOKEN token)
+{
+ return token->start == -1 ? (ANTLR3_MARKER)(token->input->data) : token->start;
+}
+
+static void setStartIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER start)
+{
+ token->start = start;
+}
+
+static ANTLR3_MARKER getStopIndex (pANTLR3_COMMON_TOKEN token)
+{
+ return token->stop;
+}
+
+static void setStopIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER stop)
+{
+ token->stop = stop;
+}
+
+static pANTLR3_STRING toString (pANTLR3_COMMON_TOKEN token)
+{
+ pANTLR3_STRING text;
+ pANTLR3_STRING outtext;
+
+ text = token->getText(token);
+
+ if (text == NULL)
+ {
+ return NULL;
+ }
+
+ if (text->factory == NULL)
+ {
+ return text; // This usall ymeans it is the EOF token
+ }
+
+ /* A new empty string to assemble all the stuff in
+ */
+ outtext = text->factory->newRaw(text->factory);
+
+ /* Now we use our handy dandy string utility to assemble the
+ * the reporting string
+ * return "[@"+getTokenIndex()+","+start+":"+stop+"='"+txt+"',<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+"]";
+ */
+ outtext->append8(outtext, "[Index: ");
+ outtext->addi (outtext, (ANTLR3_INT32)token->getTokenIndex(token));
+ outtext->append8(outtext, " (Start: ");
+ outtext->addi (outtext, (ANTLR3_INT32)token->getStartIndex(token));
+ outtext->append8(outtext, "-Stop: ");
+ outtext->addi (outtext, (ANTLR3_INT32)token->getStopIndex(token));
+ outtext->append8(outtext, ") ='");
+ outtext->appendS(outtext, text);
+ outtext->append8(outtext, "', type<");
+ outtext->addi (outtext, token->type);
+ outtext->append8(outtext, "> ");
+
+ if (token->getChannel(token) > ANTLR3_TOKEN_DEFAULT_CHANNEL)
+ {
+ outtext->append8(outtext, "(channel = ");
+ outtext->addi (outtext, (ANTLR3_INT32)token->getChannel(token));
+ outtext->append8(outtext, ") ");
+ }
+
+ outtext->append8(outtext, "Line: ");
+ outtext->addi (outtext, (ANTLR3_INT32)token->getLine(token));
+ outtext->append8(outtext, " LinePos:");
+ outtext->addi (outtext, token->getCharPositionInLine(token));
+ outtext->addc (outtext, ']');
+
+ return outtext;
+}
+
Code Review / simantics / platform.git / blobdiff