2 * Contains the default implementation of the common token used within
\r
3 * java. Custom tokens should create this structure and then append to it using the
\r
4 * custom pointer to install their own structure and API.
\r
7 // [The "BSD licence"]
\r
8 // Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
\r
9 // http://www.temporal-wave.com
\r
10 // http://www.linkedin.com/in/jimidle
\r
12 // All rights reserved.
\r
14 // Redistribution and use in source and binary forms, with or without
\r
15 // modification, are permitted provided that the following conditions
\r
17 // 1. Redistributions of source code must retain the above copyright
\r
18 // notice, this list of conditions and the following disclaimer.
\r
19 // 2. Redistributions in binary form must reproduce the above copyright
\r
20 // notice, this list of conditions and the following disclaimer in the
\r
21 // documentation and/or other materials provided with the distribution.
\r
22 // 3. The name of the author may not be used to endorse or promote products
\r
23 // derived from this software without specific prior written permission.
\r
25 // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
\r
26 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
\r
27 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
\r
28 // IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
\r
29 // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
\r
30 // NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
\r
31 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
\r
32 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
\r
33 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
\r
34 // THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\r
40 static pANTLR3_STRING getText (pANTLR3_COMMON_TOKEN token);
\r
41 static void setText (pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text);
\r
42 static void setText8 (pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text);
\r
43 static ANTLR3_UINT32 getType (pANTLR3_COMMON_TOKEN token);
\r
44 static void setType (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type);
\r
45 static ANTLR3_UINT32 getLine (pANTLR3_COMMON_TOKEN token);
\r
46 static void setLine (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line);
\r
47 static ANTLR3_INT32 getCharPositionInLine (pANTLR3_COMMON_TOKEN token);
\r
48 static void setCharPositionInLine (pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos);
\r
49 static ANTLR3_UINT32 getChannel (pANTLR3_COMMON_TOKEN token);
\r
50 static void setChannel (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel);
\r
51 static ANTLR3_MARKER getTokenIndex (pANTLR3_COMMON_TOKEN token);
\r
52 static void setTokenIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER);
\r
53 static ANTLR3_MARKER getStartIndex (pANTLR3_COMMON_TOKEN token);
\r
54 static void setStartIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);
\r
55 static ANTLR3_MARKER getStopIndex (pANTLR3_COMMON_TOKEN token);
\r
56 static void setStopIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);
\r
57 static pANTLR3_STRING toString (pANTLR3_COMMON_TOKEN token);
\r
61 static void factoryClose (pANTLR3_TOKEN_FACTORY factory);
\r
62 static pANTLR3_COMMON_TOKEN newToken (void);
\r
63 static void setInputStream (pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input);
\r
65 /* Internal management functions
\r
67 static void newPool (pANTLR3_TOKEN_FACTORY factory);
\r
68 static pANTLR3_COMMON_TOKEN newPoolToken (pANTLR3_TOKEN_FACTORY factory);
\r
72 ANTLR3_API pANTLR3_COMMON_TOKEN
\r
73 antlr3CommonTokenNew(ANTLR3_UINT32 ttype)
\r
75 pANTLR3_COMMON_TOKEN token;
\r
77 // Create a raw token with the interface installed
\r
83 token->setType(token, ttype);
\r
91 ANTLR3_API pANTLR3_TOKEN_FACTORY
\r
92 antlr3TokenFactoryNew(pANTLR3_INPUT_STREAM input)
\r
94 pANTLR3_TOKEN_FACTORY factory;
\r
98 factory = (pANTLR3_TOKEN_FACTORY) ANTLR3_MALLOC((size_t)sizeof(ANTLR3_TOKEN_FACTORY));
\r
100 if (factory == NULL)
\r
105 /* Install factory API
\r
107 factory->newToken = newPoolToken;
\r
108 factory->close = factoryClose;
\r
109 factory->setInputStream = setInputStream;
\r
111 /* Allocate the initial pool
\r
113 factory->thisPool = -1;
\r
114 factory->pools = NULL;
\r
117 /* Factory space is good, we now want to initialize our cheating token
\r
118 * which one it is initialized is the model for all tokens we manufacture
\r
120 antlr3SetTokenAPI(&factory->unTruc);
\r
122 /* Set some initial variables for future copying
\r
124 factory->unTruc.factoryMade = ANTLR3_TRUE;
\r
128 setInputStream(factory, input);
\r
135 setInputStream (pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input)
\r
137 factory->input = input;
\r
138 factory->unTruc.input = input;
\r
141 factory->unTruc.strFactory = input->strFactory;
\r
145 factory->unTruc.strFactory = NULL;
\r
150 newPool(pANTLR3_TOKEN_FACTORY factory)
\r
152 /* Increment factory count
\r
154 factory->thisPool++;
\r
156 /* Ensure we have enough pointers allocated
\r
158 factory->pools = (pANTLR3_COMMON_TOKEN *)
\r
159 ANTLR3_REALLOC( (void *)factory->pools, /* Current pools pointer (starts at NULL) */
\r
160 (ANTLR3_UINT32)((factory->thisPool + 1) * sizeof(pANTLR3_COMMON_TOKEN *)) /* Memory for new pool pointers */
\r
163 /* Allocate a new pool for the factory
\r
165 factory->pools[factory->thisPool] =
\r
166 (pANTLR3_COMMON_TOKEN)
\r
167 ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_COMMON_TOKEN) * ANTLR3_FACTORY_POOL_SIZE));
\r
169 /* Reset the counters
\r
171 factory->nextToken = 0;
\r
178 static pANTLR3_COMMON_TOKEN
\r
179 newPoolToken(pANTLR3_TOKEN_FACTORY factory)
\r
181 pANTLR3_COMMON_TOKEN token;
\r
183 /* See if we need a new token pool before allocating a new
\r
186 if (factory->nextToken >= ANTLR3_FACTORY_POOL_SIZE)
\r
188 /* We ran out of tokens in the current pool, so we need a new pool
\r
193 /* Assuming everything went well (we are trying for performance here so doing minimal
\r
194 * error checking. Then we can work out what the pointer is to the next token.
\r
196 token = factory->pools[factory->thisPool] + factory->nextToken;
\r
197 factory->nextToken++;
\r
199 /* We have our token pointer now, so we can initialize it to the predefined model.
\r
201 antlr3SetTokenAPI(token);
\r
203 /* It is factory made, and we need to copy the string factory pointer
\r
205 token->factoryMade = ANTLR3_TRUE;
\r
206 token->strFactory = factory->input == NULL ? NULL : factory->input->strFactory;
\r
207 token->input = factory->input;
\r
215 factoryClose (pANTLR3_TOKEN_FACTORY factory)
\r
217 pANTLR3_COMMON_TOKEN pool;
\r
218 ANTLR3_INT32 poolCount;
\r
219 ANTLR3_UINT32 limit;
\r
220 ANTLR3_UINT32 token;
\r
221 pANTLR3_COMMON_TOKEN check;
\r
223 /* We iterate the token pools one at a time
\r
225 for (poolCount = 0; poolCount <= factory->thisPool; poolCount++)
\r
227 /* Pointer to current pool
\r
229 pool = factory->pools[poolCount];
\r
231 /* Work out how many tokens we need to check in this pool.
\r
233 limit = (poolCount == factory->thisPool ? factory->nextToken : ANTLR3_FACTORY_POOL_SIZE);
\r
235 /* Marginal condition, we might be at the start of a brand new pool
\r
236 * where the nextToken is 0 and nothing has been allocated.
\r
240 /* We have some tokens allocated from this pool
\r
242 for (token = 0; token < limit; token++)
\r
244 /* Next one in the chain
\r
246 check = pool + token;
\r
248 /* If the programmer made this a custom token, then
\r
249 * see if we need to call their free routine.
\r
251 if (check->custom != NULL && check->freeCustom != NULL)
\r
253 check->freeCustom(check->custom);
\r
254 check->custom = NULL;
\r
259 /* We can now free this pool allocation
\r
261 ANTLR3_FREE(factory->pools[poolCount]);
\r
262 factory->pools[poolCount] = NULL;
\r
265 /* All the pools are deallocated we can free the pointers to the pools
\r
268 ANTLR3_FREE(factory->pools);
\r
270 /* Finally, we can free the space for the factory itself
\r
272 ANTLR3_FREE(factory);
\r
276 static pANTLR3_COMMON_TOKEN
\r
279 pANTLR3_COMMON_TOKEN token;
\r
281 /* Allocate memory for this
\r
283 token = (pANTLR3_COMMON_TOKEN) ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_COMMON_TOKEN)));
\r
292 antlr3SetTokenAPI(token);
\r
293 token->factoryMade = ANTLR3_FALSE;
\r
299 antlr3SetTokenAPI(pANTLR3_COMMON_TOKEN token)
\r
301 token->getText = getText;
\r
302 token->setText = setText;
\r
303 token->setText8 = setText8;
\r
304 token->getType = getType;
\r
305 token->setType = setType;
\r
306 token->getLine = getLine;
\r
307 token->setLine = setLine;
\r
308 token->setLine = setLine;
\r
309 token->getCharPositionInLine = getCharPositionInLine;
\r
310 token->setCharPositionInLine = setCharPositionInLine;
\r
311 token->getChannel = getChannel;
\r
312 token->setChannel = setChannel;
\r
313 token->getTokenIndex = getTokenIndex;
\r
314 token->setTokenIndex = setTokenIndex;
\r
315 token->getStartIndex = getStartIndex;
\r
316 token->setStartIndex = setStartIndex;
\r
317 token->getStopIndex = getStopIndex;
\r
318 token->setStopIndex = setStopIndex;
\r
319 token->toString = toString;
\r
323 token->setCharPositionInLine(token, -1);
\r
325 token->custom = NULL;
\r
326 token->freeCustom = NULL;
\r
327 token->type = ANTLR3_TOKEN_INVALID;
\r
328 token->textState = ANTLR3_TEXT_NONE;
\r
331 token->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;
\r
334 token->input = NULL;
\r
338 token->custom = NULL;
\r
343 static pANTLR3_STRING getText (pANTLR3_COMMON_TOKEN token)
\r
345 switch (token->textState)
\r
347 case ANTLR3_TEXT_STRING:
\r
349 // Someone already created a string for this token, so we just
\r
352 return token->tokText.text;
\r
355 case ANTLR3_TEXT_CHARP:
\r
357 // We had a straight text pointer installed, now we
\r
358 // must convert it to a string. Note we have to do this here
\r
359 // or otherwise setText8() will just install the same char*
\r
361 if (token->strFactory != NULL)
\r
363 token->tokText.text = token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)token->tokText.chars);
\r
364 token->textState = ANTLR3_TEXT_STRING;
\r
365 return token->tokText.text;
\r
369 // We cannot do anything here
\r
377 // EOF is a special case
\r
379 if (token->type == ANTLR3_TOKEN_EOF)
\r
381 token->tokText.text = token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)"<EOF>");
\r
382 token->textState = ANTLR3_TEXT_STRING;
\r
383 return token->tokText.text;
\r
387 // We had nothing installed in the token, create a new string
\r
388 // from the input stream
\r
391 if (token->input != NULL)
\r
394 return token->input->substr( token->input,
\r
395 token->getStartIndex(token),
\r
396 token->getStopIndex(token)
\r
400 // Nothing to return, there is no input stream
\r
406 static void setText8 (pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text)
\r
408 // No text to set, so ignore
\r
410 if (text == NULL) return;
\r
412 switch (token->textState)
\r
414 case ANTLR3_TEXT_NONE:
\r
415 case ANTLR3_TEXT_CHARP: // Caller must free before setting again, if it needs to be freed
\r
417 // Nothing in there yet, or just a char *, so just set the
\r
418 // text as a pointer
\r
420 token->textState = ANTLR3_TEXT_CHARP;
\r
421 token->tokText.chars = (pANTLR3_UCHAR)text;
\r
426 // It was already a pANTLR3_STRING, so just override it
\r
428 token->tokText.text->set8(token->tokText.text, (const char *)text);
\r
437 /** \brief Install the supplied text string as teh text for the token.
\r
438 * The method assumes that the existing text (if any) was created by a factory
\r
439 * and so does not attempt to release any memory it is using.Text not created
\r
440 * by a string fctory (not advised) should be released prior to this call.
\r
442 static void setText (pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text)
\r
444 // Merely replaces and existing pre-defined text with the supplied
\r
447 token->textState = ANTLR3_TEXT_STRING;
\r
448 token->tokText.text = text;
\r
455 static ANTLR3_UINT32 getType (pANTLR3_COMMON_TOKEN token)
\r
457 return token->type;
\r
460 static void setType (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type)
\r
462 token->type = type;
\r
465 static ANTLR3_UINT32 getLine (pANTLR3_COMMON_TOKEN token)
\r
467 return token->line;
\r
470 static void setLine (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line)
\r
472 token->line = line;
\r
475 static ANTLR3_INT32 getCharPositionInLine (pANTLR3_COMMON_TOKEN token)
\r
477 return token->charPosition;
\r
480 static void setCharPositionInLine (pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos)
\r
482 token->charPosition = pos;
\r
485 static ANTLR3_UINT32 getChannel (pANTLR3_COMMON_TOKEN token)
\r
487 return token->channel;
\r
490 static void setChannel (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel)
\r
492 token->channel = channel;
\r
495 static ANTLR3_MARKER getTokenIndex (pANTLR3_COMMON_TOKEN token)
\r
497 return token->index;
\r
500 static void setTokenIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index)
\r
502 token->index = index;
\r
505 static ANTLR3_MARKER getStartIndex (pANTLR3_COMMON_TOKEN token)
\r
507 return token->start == -1 ? (ANTLR3_MARKER)(token->input->data) : token->start;
\r
510 static void setStartIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER start)
\r
512 token->start = start;
\r
515 static ANTLR3_MARKER getStopIndex (pANTLR3_COMMON_TOKEN token)
\r
517 return token->stop;
\r
520 static void setStopIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER stop)
\r
522 token->stop = stop;
\r
525 static pANTLR3_STRING toString (pANTLR3_COMMON_TOKEN token)
\r
527 pANTLR3_STRING text;
\r
528 pANTLR3_STRING outtext;
\r
530 text = token->getText(token);
\r
537 if (text->factory == NULL)
\r
539 return text; // This usall ymeans it is the EOF token
\r
542 /* A new empty string to assemble all the stuff in
\r
544 outtext = text->factory->newRaw(text->factory);
\r
546 /* Now we use our handy dandy string utility to assemble the
\r
547 * the reporting string
\r
548 * return "[@"+getTokenIndex()+","+start+":"+stop+"='"+txt+"',<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+"]";
\r
550 outtext->append8(outtext, "[Index: ");
\r
551 outtext->addi (outtext, (ANTLR3_INT32)token->getTokenIndex(token));
\r
552 outtext->append8(outtext, " (Start: ");
\r
553 outtext->addi (outtext, (ANTLR3_INT32)token->getStartIndex(token));
\r
554 outtext->append8(outtext, "-Stop: ");
\r
555 outtext->addi (outtext, (ANTLR3_INT32)token->getStopIndex(token));
\r
556 outtext->append8(outtext, ") ='");
\r
557 outtext->appendS(outtext, text);
\r
558 outtext->append8(outtext, "', type<");
\r
559 outtext->addi (outtext, token->type);
\r
560 outtext->append8(outtext, "> ");
\r
562 if (token->getChannel(token) > ANTLR3_TOKEN_DEFAULT_CHANNEL)
\r
564 outtext->append8(outtext, "(channel = ");
\r
565 outtext->addi (outtext, (ANTLR3_INT32)token->getChannel(token));
\r
566 outtext->append8(outtext, ") ");
\r
569 outtext->append8(outtext, "Line: ");
\r
570 outtext->addi (outtext, (ANTLR3_INT32)token->getLine(token));
\r
571 outtext->append8(outtext, " LinePos:");
\r
572 outtext->addi (outtext, token->getCharPositionInLine(token));
\r
573 outtext->addc (outtext, ']');
\r