2 * Contains the default implementation of the common token used within
3 * java. Custom tokens should create this structure and then append to it using the
4 * custom pointer to install their own structure and API.
8 // Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
9 // http://www.temporal-wave.com
10 // http://www.linkedin.com/in/jimidle
12 // All rights reserved.
14 // Redistribution and use in source and binary forms, with or without
15 // modification, are permitted provided that the following conditions
17 // 1. Redistributions of source code must retain the above copyright
18 // notice, this list of conditions and the following disclaimer.
19 // 2. Redistributions in binary form must reproduce the above copyright
20 // notice, this list of conditions and the following disclaimer in the
21 // documentation and/or other materials provided with the distribution.
22 // 3. The name of the author may not be used to endorse or promote products
23 // derived from this software without specific prior written permission.
25 // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
26 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
27 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
28 // IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
29 // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
30 // NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
31 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
32 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
33 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
34 // THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40 static pANTLR3_STRING getText (pANTLR3_COMMON_TOKEN token);
41 static void setText (pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text);
42 static void setText8 (pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text);
43 static ANTLR3_UINT32 getType (pANTLR3_COMMON_TOKEN token);
44 static void setType (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type);
45 static ANTLR3_UINT32 getLine (pANTLR3_COMMON_TOKEN token);
46 static void setLine (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line);
47 static ANTLR3_INT32 getCharPositionInLine (pANTLR3_COMMON_TOKEN token);
48 static void setCharPositionInLine (pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos);
49 static ANTLR3_UINT32 getChannel (pANTLR3_COMMON_TOKEN token);
50 static void setChannel (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel);
51 static ANTLR3_MARKER getTokenIndex (pANTLR3_COMMON_TOKEN token);
52 static void setTokenIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER);
53 static ANTLR3_MARKER getStartIndex (pANTLR3_COMMON_TOKEN token);
54 static void setStartIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);
55 static ANTLR3_MARKER getStopIndex (pANTLR3_COMMON_TOKEN token);
56 static void setStopIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);
57 static pANTLR3_STRING toString (pANTLR3_COMMON_TOKEN token);
61 static void factoryClose (pANTLR3_TOKEN_FACTORY factory);
62 static pANTLR3_COMMON_TOKEN newToken (void);
63 static void setInputStream (pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input);
65 /* Internal management functions
67 static void newPool (pANTLR3_TOKEN_FACTORY factory);
68 static pANTLR3_COMMON_TOKEN newPoolToken (pANTLR3_TOKEN_FACTORY factory);
72 ANTLR3_API pANTLR3_COMMON_TOKEN
73 antlr3CommonTokenNew(ANTLR3_UINT32 ttype)
75 pANTLR3_COMMON_TOKEN token;
77 // Create a raw token with the interface installed
83 token->setType(token, ttype);
91 ANTLR3_API pANTLR3_TOKEN_FACTORY
92 antlr3TokenFactoryNew(pANTLR3_INPUT_STREAM input)
94 pANTLR3_TOKEN_FACTORY factory;
98 factory = (pANTLR3_TOKEN_FACTORY) ANTLR3_MALLOC((size_t)sizeof(ANTLR3_TOKEN_FACTORY));
105 /* Install factory API
107 factory->newToken = newPoolToken;
108 factory->close = factoryClose;
109 factory->setInputStream = setInputStream;
111 /* Allocate the initial pool
113 factory->thisPool = -1;
114 factory->pools = NULL;
117 /* Factory space is good, we now want to initialize our cheating token
118 * which one it is initialized is the model for all tokens we manufacture
120 antlr3SetTokenAPI(&factory->unTruc);
122 /* Set some initial variables for future copying
124 factory->unTruc.factoryMade = ANTLR3_TRUE;
128 setInputStream(factory, input);
135 setInputStream (pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input)
137 factory->input = input;
138 factory->unTruc.input = input;
141 factory->unTruc.strFactory = input->strFactory;
145 factory->unTruc.strFactory = NULL;
150 newPool(pANTLR3_TOKEN_FACTORY factory)
152 /* Increment factory count
156 /* Ensure we have enough pointers allocated
158 factory->pools = (pANTLR3_COMMON_TOKEN *)
159 ANTLR3_REALLOC( (void *)factory->pools, /* Current pools pointer (starts at NULL) */
160 (ANTLR3_UINT32)((factory->thisPool + 1) * sizeof(pANTLR3_COMMON_TOKEN *)) /* Memory for new pool pointers */
163 /* Allocate a new pool for the factory
165 factory->pools[factory->thisPool] =
166 (pANTLR3_COMMON_TOKEN)
167 ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_COMMON_TOKEN) * ANTLR3_FACTORY_POOL_SIZE));
169 /* Reset the counters
171 factory->nextToken = 0;
178 static pANTLR3_COMMON_TOKEN
179 newPoolToken(pANTLR3_TOKEN_FACTORY factory)
181 pANTLR3_COMMON_TOKEN token;
183 /* See if we need a new token pool before allocating a new
186 if (factory->nextToken >= ANTLR3_FACTORY_POOL_SIZE)
188 /* We ran out of tokens in the current pool, so we need a new pool
193 /* Assuming everything went well (we are trying for performance here so doing minimal
194 * error checking. Then we can work out what the pointer is to the next token.
196 token = factory->pools[factory->thisPool] + factory->nextToken;
197 factory->nextToken++;
199 /* We have our token pointer now, so we can initialize it to the predefined model.
201 antlr3SetTokenAPI(token);
203 /* It is factory made, and we need to copy the string factory pointer
205 token->factoryMade = ANTLR3_TRUE;
206 token->strFactory = factory->input == NULL ? NULL : factory->input->strFactory;
207 token->input = factory->input;
215 factoryClose (pANTLR3_TOKEN_FACTORY factory)
217 pANTLR3_COMMON_TOKEN pool;
218 ANTLR3_INT32 poolCount;
221 pANTLR3_COMMON_TOKEN check;
223 /* We iterate the token pools one at a time
225 for (poolCount = 0; poolCount <= factory->thisPool; poolCount++)
227 /* Pointer to current pool
229 pool = factory->pools[poolCount];
231 /* Work out how many tokens we need to check in this pool.
233 limit = (poolCount == factory->thisPool ? factory->nextToken : ANTLR3_FACTORY_POOL_SIZE);
235 /* Marginal condition, we might be at the start of a brand new pool
236 * where the nextToken is 0 and nothing has been allocated.
240 /* We have some tokens allocated from this pool
242 for (token = 0; token < limit; token++)
244 /* Next one in the chain
246 check = pool + token;
248 /* If the programmer made this a custom token, then
249 * see if we need to call their free routine.
251 if (check->custom != NULL && check->freeCustom != NULL)
253 check->freeCustom(check->custom);
254 check->custom = NULL;
259 /* We can now free this pool allocation
261 ANTLR3_FREE(factory->pools[poolCount]);
262 factory->pools[poolCount] = NULL;
265 /* All the pools are deallocated we can free the pointers to the pools
268 ANTLR3_FREE(factory->pools);
270 /* Finally, we can free the space for the factory itself
272 ANTLR3_FREE(factory);
276 static pANTLR3_COMMON_TOKEN
279 pANTLR3_COMMON_TOKEN token;
281 /* Allocate memory for this
283 token = (pANTLR3_COMMON_TOKEN) ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_COMMON_TOKEN)));
292 antlr3SetTokenAPI(token);
293 token->factoryMade = ANTLR3_FALSE;
299 antlr3SetTokenAPI(pANTLR3_COMMON_TOKEN token)
301 token->getText = getText;
302 token->setText = setText;
303 token->setText8 = setText8;
304 token->getType = getType;
305 token->setType = setType;
306 token->getLine = getLine;
307 token->setLine = setLine;
308 token->setLine = setLine;
309 token->getCharPositionInLine = getCharPositionInLine;
310 token->setCharPositionInLine = setCharPositionInLine;
311 token->getChannel = getChannel;
312 token->setChannel = setChannel;
313 token->getTokenIndex = getTokenIndex;
314 token->setTokenIndex = setTokenIndex;
315 token->getStartIndex = getStartIndex;
316 token->setStartIndex = setStartIndex;
317 token->getStopIndex = getStopIndex;
318 token->setStopIndex = setStopIndex;
319 token->toString = toString;
323 token->setCharPositionInLine(token, -1);
325 token->custom = NULL;
326 token->freeCustom = NULL;
327 token->type = ANTLR3_TOKEN_INVALID;
328 token->textState = ANTLR3_TEXT_NONE;
331 token->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;
338 token->custom = NULL;
343 static pANTLR3_STRING getText (pANTLR3_COMMON_TOKEN token)
345 switch (token->textState)
347 case ANTLR3_TEXT_STRING:
349 // Someone already created a string for this token, so we just
352 return token->tokText.text;
355 case ANTLR3_TEXT_CHARP:
357 // We had a straight text pointer installed, now we
358 // must convert it to a string. Note we have to do this here
359 // or otherwise setText8() will just install the same char*
361 if (token->strFactory != NULL)
363 token->tokText.text = token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)token->tokText.chars);
364 token->textState = ANTLR3_TEXT_STRING;
365 return token->tokText.text;
369 // We cannot do anything here
377 // EOF is a special case
379 if (token->type == ANTLR3_TOKEN_EOF)
381 token->tokText.text = token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)"<EOF>");
382 token->textState = ANTLR3_TEXT_STRING;
383 return token->tokText.text;
387 // We had nothing installed in the token, create a new string
388 // from the input stream
391 if (token->input != NULL)
394 return token->input->substr( token->input,
395 token->getStartIndex(token),
396 token->getStopIndex(token)
400 // Nothing to return, there is no input stream
406 static void setText8 (pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text)
408 // No text to set, so ignore
410 if (text == NULL) return;
412 switch (token->textState)
414 case ANTLR3_TEXT_NONE:
415 case ANTLR3_TEXT_CHARP: // Caller must free before setting again, if it needs to be freed
417 // Nothing in there yet, or just a char *, so just set the
420 token->textState = ANTLR3_TEXT_CHARP;
421 token->tokText.chars = (pANTLR3_UCHAR)text;
426 // It was already a pANTLR3_STRING, so just override it
428 token->tokText.text->set8(token->tokText.text, (const char *)text);
437 /** \brief Install the supplied text string as teh text for the token.
438 * The method assumes that the existing text (if any) was created by a factory
439 * and so does not attempt to release any memory it is using.Text not created
440 * by a string fctory (not advised) should be released prior to this call.
442 static void setText (pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text)
444 // Merely replaces and existing pre-defined text with the supplied
447 token->textState = ANTLR3_TEXT_STRING;
448 token->tokText.text = text;
455 static ANTLR3_UINT32 getType (pANTLR3_COMMON_TOKEN token)
460 static void setType (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type)
465 static ANTLR3_UINT32 getLine (pANTLR3_COMMON_TOKEN token)
470 static void setLine (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line)
475 static ANTLR3_INT32 getCharPositionInLine (pANTLR3_COMMON_TOKEN token)
477 return token->charPosition;
480 static void setCharPositionInLine (pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos)
482 token->charPosition = pos;
485 static ANTLR3_UINT32 getChannel (pANTLR3_COMMON_TOKEN token)
487 return token->channel;
490 static void setChannel (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel)
492 token->channel = channel;
495 static ANTLR3_MARKER getTokenIndex (pANTLR3_COMMON_TOKEN token)
500 static void setTokenIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index)
502 token->index = index;
505 static ANTLR3_MARKER getStartIndex (pANTLR3_COMMON_TOKEN token)
507 return token->start == -1 ? (ANTLR3_MARKER)(token->input->data) : token->start;
510 static void setStartIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER start)
512 token->start = start;
515 static ANTLR3_MARKER getStopIndex (pANTLR3_COMMON_TOKEN token)
520 static void setStopIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER stop)
525 static pANTLR3_STRING toString (pANTLR3_COMMON_TOKEN token)
528 pANTLR3_STRING outtext;
530 text = token->getText(token);
537 if (text->factory == NULL)
539 return text; // This usall ymeans it is the EOF token
542 /* A new empty string to assemble all the stuff in
544 outtext = text->factory->newRaw(text->factory);
546 /* Now we use our handy dandy string utility to assemble the
547 * the reporting string
548 * return "[@"+getTokenIndex()+","+start+":"+stop+"='"+txt+"',<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+"]";
550 outtext->append8(outtext, "[Index: ");
551 outtext->addi (outtext, (ANTLR3_INT32)token->getTokenIndex(token));
552 outtext->append8(outtext, " (Start: ");
553 outtext->addi (outtext, (ANTLR3_INT32)token->getStartIndex(token));
554 outtext->append8(outtext, "-Stop: ");
555 outtext->addi (outtext, (ANTLR3_INT32)token->getStopIndex(token));
556 outtext->append8(outtext, ") ='");
557 outtext->appendS(outtext, text);
558 outtext->append8(outtext, "', type<");
559 outtext->addi (outtext, token->type);
560 outtext->append8(outtext, "> ");
562 if (token->getChannel(token) > ANTLR3_TOKEN_DEFAULT_CHANNEL)
564 outtext->append8(outtext, "(channel = ");
565 outtext->addi (outtext, (ANTLR3_INT32)token->getChannel(token));
566 outtext->append8(outtext, ") ");
569 outtext->append8(outtext, "Line: ");
570 outtext->addi (outtext, (ANTLR3_INT32)token->getLine(token));
571 outtext->append8(outtext, " LinePos:");
572 outtext->addi (outtext, token->getCharPositionInLine(token));
573 outtext->addc (outtext, ']');