2 /// Default implementation of CommonTokenStream
\r
5 // [The "BSD licence"]
\r
6 // Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
\r
7 // http://www.temporal-wave.com
\r
8 // http://www.linkedin.com/in/jimidle
\r
10 // All rights reserved.
\r
12 // Redistribution and use in source and binary forms, with or without
\r
13 // modification, are permitted provided that the following conditions
\r
15 // 1. Redistributions of source code must retain the above copyright
\r
16 // notice, this list of conditions and the following disclaimer.
\r
17 // 2. Redistributions in binary form must reproduce the above copyright
\r
18 // notice, this list of conditions and the following disclaimer in the
\r
19 // documentation and/or other materials provided with the distribution.
\r
20 // 3. The name of the author may not be used to endorse or promote products
\r
21 // derived from this software without specific prior written permission.
\r
23 // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
\r
24 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
\r
25 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
\r
26 // IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
\r
27 // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
\r
28 // NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
\r
29 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
\r
30 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
\r
31 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
\r
32 // THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\r
34 #include <antlr3tokenstream.h>
\r
36 #ifdef ANTLR3_WINDOWS
\r
37 #pragma warning( disable : 4100 )
\r
40 // COMMON_TOKEN_STREAM API
\r
42 static void setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel);
\r
43 static void discardTokenType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 ttype);
\r
44 static void discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_BOOLEAN discard);
\r
45 static pANTLR3_VECTOR getTokens (pANTLR3_COMMON_TOKEN_STREAM cts);
\r
46 static pANTLR3_LIST getTokenRange (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
\r
47 static pANTLR3_LIST getTokensSet (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types);
\r
48 static pANTLR3_LIST getTokensList (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list);
\r
49 static pANTLR3_LIST getTokensType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type);
\r
51 // TOKEN_STREAM API
\r
53 static pANTLR3_COMMON_TOKEN tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);
\r
54 static pANTLR3_COMMON_TOKEN dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);
\r
55 static pANTLR3_COMMON_TOKEN get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i);
\r
56 static pANTLR3_TOKEN_SOURCE getTokenSource (pANTLR3_TOKEN_STREAM ts);
\r
57 static void setTokenSource (pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource);
\r
58 static pANTLR3_STRING toString (pANTLR3_TOKEN_STREAM ts);
\r
59 static pANTLR3_STRING toStringSS (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
\r
60 static pANTLR3_STRING toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop);
\r
61 static void setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger);
\r
65 static void consume (pANTLR3_INT_STREAM is);
\r
66 static void dbgConsume (pANTLR3_INT_STREAM is);
\r
67 static ANTLR3_UINT32 _LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i);
\r
68 static ANTLR3_UINT32 dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i);
\r
69 static ANTLR3_MARKER mark (pANTLR3_INT_STREAM is);
\r
70 static ANTLR3_MARKER dbgMark (pANTLR3_INT_STREAM is);
\r
71 static void release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark);
\r
72 static ANTLR3_UINT32 size (pANTLR3_INT_STREAM is);
\r
73 static ANTLR3_MARKER tindex (pANTLR3_INT_STREAM is);
\r
74 static void rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);
\r
75 static void dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);
\r
76 static void rewindLast (pANTLR3_INT_STREAM is);
\r
77 static void dbgRewindLast (pANTLR3_INT_STREAM is);
\r
78 static void seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index);
\r
79 static void dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index);
\r
80 static pANTLR3_STRING getSourceName (pANTLR3_INT_STREAM is);
\r
81 static void antlr3TokenStreamFree (pANTLR3_TOKEN_STREAM stream);
\r
82 static void antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream);
\r
86 static void fillBuffer (pANTLR3_COMMON_TOKEN_STREAM tokenStream);
\r
87 static ANTLR3_UINT32 skipOffTokenChannels (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
\r
88 static ANTLR3_UINT32 skipOffTokenChannelsReverse (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
\r
89 static pANTLR3_COMMON_TOKEN LB (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
\r
91 ANTLR3_API pANTLR3_TOKEN_STREAM
\r
92 antlr3TokenStreamNew()
\r
94 pANTLR3_TOKEN_STREAM stream;
\r
96 // Memory for the interface structure
\r
98 stream = (pANTLR3_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_STREAM));
\r
100 if (stream == NULL)
\r
105 // Install basic API
\r
107 stream->free = antlr3TokenStreamFree;
\r
114 antlr3TokenStreamFree(pANTLR3_TOKEN_STREAM stream)
\r
116 ANTLR3_FREE(stream);
\r
120 antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream)
\r
122 // We only free up our subordinate interfaces if they belong
\r
123 // to us, otherwise we let whoever owns them deal with them.
\r
125 if (stream->tstream->super == stream)
\r
127 if (stream->tstream->istream->super == stream->tstream)
\r
129 stream->tstream->istream->free(stream->tstream->istream);
\r
130 stream->tstream->istream = NULL;
\r
132 stream->tstream->free(stream->tstream);
\r
135 // Now we free our own resources
\r
137 if (stream->tokens != NULL)
\r
139 stream->tokens->free(stream->tokens);
\r
140 stream->tokens = NULL;
\r
142 if (stream->discardSet != NULL)
\r
144 stream->discardSet->free(stream->discardSet);
\r
145 stream->discardSet = NULL;
\r
147 if (stream->channelOverrides != NULL)
\r
149 stream->channelOverrides->free(stream->channelOverrides);
\r
150 stream->channelOverrides = NULL;
\r
153 // Free our memory now
\r
155 ANTLR3_FREE(stream);
\r
158 ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
\r
159 antlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger)
\r
161 pANTLR3_COMMON_TOKEN_STREAM stream;
\r
163 // Create a standard token stream
\r
165 stream = antlr3CommonTokenStreamSourceNew(hint, source);
\r
167 // Install the debugger object
\r
169 stream->tstream->debugger = debugger;
\r
171 // Override standard token stream methods with debugging versions
\r
173 stream->tstream->initialStreamState = ANTLR3_FALSE;
\r
175 stream->tstream->_LT = dbgTokLT;
\r
177 stream->tstream->istream->consume = dbgConsume;
\r
178 stream->tstream->istream->_LA = dbgLA;
\r
179 stream->tstream->istream->mark = dbgMark;
\r
180 stream->tstream->istream->rewind = dbgRewindStream;
\r
181 stream->tstream->istream->rewindLast = dbgRewindLast;
\r
182 stream->tstream->istream->seek = dbgSeek;
\r
187 ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
\r
188 antlr3CommonTokenStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source)
\r
190 pANTLR3_COMMON_TOKEN_STREAM stream;
\r
192 stream = antlr3CommonTokenStreamNew(hint);
\r
194 stream->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;
\r
196 stream->channelOverrides = NULL;
\r
197 stream->discardSet = NULL;
\r
198 stream->discardOffChannel = ANTLR3_FALSE;
\r
200 stream->tstream->setTokenSource(stream->tstream, source);
\r
202 stream->free = antlr3CTSFree;
\r
206 ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
\r
207 antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint)
\r
209 pANTLR3_COMMON_TOKEN_STREAM stream;
\r
211 /* Memory for the interface structure
\r
213 stream = (pANTLR3_COMMON_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TOKEN_STREAM));
\r
215 if (stream == NULL)
\r
220 /* Create space for the token stream interface
\r
222 stream->tstream = antlr3TokenStreamNew();
\r
223 stream->tstream->super = stream;
\r
225 /* Create space for the INT_STREAM interfacce
\r
227 stream->tstream->istream = antlr3IntStreamNew();
\r
228 stream->tstream->istream->super = (stream->tstream);
\r
229 stream->tstream->istream->type = ANTLR3_TOKENSTREAM;
\r
231 /* Install the token tracking tables
\r
233 stream->tokens = antlr3VectorNew(0);
\r
239 /* Install the common token stream API
\r
241 stream->setTokenTypeChannel = setTokenTypeChannel;
\r
242 stream->discardTokenType = discardTokenType;
\r
243 stream->discardOffChannelToks = discardOffChannel;
\r
244 stream->getTokens = getTokens;
\r
245 stream->getTokenRange = getTokenRange;
\r
246 stream->getTokensSet = getTokensSet;
\r
247 stream->getTokensList = getTokensList;
\r
248 stream->getTokensType = getTokensType;
\r
250 /* Install the token stream API
\r
252 stream->tstream->_LT = tokLT;
\r
253 stream->tstream->get = get;
\r
254 stream->tstream->getTokenSource = getTokenSource;
\r
255 stream->tstream->setTokenSource = setTokenSource;
\r
256 stream->tstream->toString = toString;
\r
257 stream->tstream->toStringSS = toStringSS;
\r
258 stream->tstream->toStringTT = toStringTT;
\r
259 stream->tstream->setDebugListener = setDebugListener;
\r
261 /* Install INT_STREAM interface
\r
263 stream->tstream->istream->_LA = _LA;
\r
264 stream->tstream->istream->mark = mark;
\r
265 stream->tstream->istream->release = release;
\r
266 stream->tstream->istream->size = size;
\r
267 stream->tstream->istream->index = tindex;
\r
268 stream->tstream->istream->rewind = rewindStream;
\r
269 stream->tstream->istream->rewindLast= rewindLast;
\r
270 stream->tstream->istream->seek = seek;
\r
271 stream->tstream->istream->consume = consume;
\r
272 stream->tstream->istream->getSourceName = getSourceName;
\r
277 // Install a debug listener adn switch to debug mode methods
\r
280 setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger)
\r
282 // Install the debugger object
\r
284 ts->debugger = debugger;
\r
286 // Override standard token stream methods with debugging versions
\r
288 ts->initialStreamState = ANTLR3_FALSE;
\r
290 ts->_LT = dbgTokLT;
\r
292 ts->istream->consume = dbgConsume;
\r
293 ts->istream->_LA = dbgLA;
\r
294 ts->istream->mark = dbgMark;
\r
295 ts->istream->rewind = dbgRewindStream;
\r
296 ts->istream->rewindLast = dbgRewindLast;
\r
297 ts->istream->seek = dbgSeek;
\r
300 /** Get the ith token from the current position 1..n where k=1 is the
\r
301 * first symbol of lookahead.
\r
303 static pANTLR3_COMMON_TOKEN
\r
304 tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
\r
308 pANTLR3_COMMON_TOKEN_STREAM cts;
\r
310 cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
\r
314 return LB(cts, -k);
\r
326 if ((cts->p + k - 1) >= (ANTLR3_INT32)ts->istream->cachedSize)
\r
328 pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken);
\r
330 teof->setStartIndex (teof, ts->istream->index (ts->istream));
\r
331 teof->setStopIndex (teof, ts->istream->index (ts->istream));
\r
338 /* Need to find k good tokens, skipping ones that are off channel
\r
342 /* Skip off-channel tokens */
\r
343 i = skipOffTokenChannels(cts, i+1); /* leave p on valid token */
\r
346 if ( (ANTLR3_UINT32) i >= ts->istream->cachedSize)
\r
348 pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken);
\r
350 teof->setStartIndex (teof, ts->istream->index(ts->istream));
\r
351 teof->setStopIndex (teof, ts->istream->index(ts->istream));
\r
355 // Here the token must be in the input vector. Rather then incut
\r
356 // function call penalty, we jsut return the pointer directly
\r
359 return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;
\r
360 //return (pANTLR3_COMMON_TOKEN)cts->tokens->get(cts->tokens, i);
\r
363 /// Debug only method to flag consumption of initial off-channel
\r
364 /// tokens in the input stream
\r
367 consumeInitialHiddenTokens(pANTLR3_INT_STREAM is)
\r
369 ANTLR3_MARKER first;
\r
371 pANTLR3_TOKEN_STREAM ts;
\r
373 ts = (pANTLR3_TOKEN_STREAM) is->super;
\r
374 first = is->index(is);
\r
376 for (i=0; i<first; i++)
\r
378 ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, i));
\r
381 ts->initialStreamState = ANTLR3_FALSE;
\r
385 /// As per the normal tokLT but sends information to the debugger
\r
387 static pANTLR3_COMMON_TOKEN
\r
388 dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
\r
390 if (ts->initialStreamState == ANTLR3_TRUE)
\r
392 consumeInitialHiddenTokens(ts->istream);
\r
394 return tokLT(ts, k);
\r
397 #ifdef ANTLR3_WINDOWS
\r
398 /* When fully optimized VC7 complains about non reachable code.
\r
399 * Not yet sure if this is an optimizer bug, or a bug in the flow analysis
\r
401 #pragma warning( disable : 4702 )
\r
404 static pANTLR3_COMMON_TOKEN
\r
405 LB(pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 k)
\r
418 if ((cts->p - k) < 0)
\r
426 /* Need to find k good tokens, going backwards, skipping ones that are off channel
\r
428 while (n <= (ANTLR3_INT32) k)
\r
430 /* Skip off-channel tokens
\r
433 i = skipOffTokenChannelsReverse(cts, i - 1); /* leave p on valid token */
\r
440 // Here the token must be in the input vector. Rather then incut
\r
441 // function call penalty, we jsut return the pointer directly
\r
444 return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;
\r
447 static pANTLR3_COMMON_TOKEN
\r
448 get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i)
\r
450 pANTLR3_COMMON_TOKEN_STREAM cts;
\r
452 cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
\r
454 return (pANTLR3_COMMON_TOKEN)(cts->tokens->get(cts->tokens, i)); /* Token index is zero based but vectors are 1 based */
\r
457 static pANTLR3_TOKEN_SOURCE
\r
458 getTokenSource (pANTLR3_TOKEN_STREAM ts)
\r
460 return ts->tokenSource;
\r
464 setTokenSource ( pANTLR3_TOKEN_STREAM ts,
\r
465 pANTLR3_TOKEN_SOURCE tokenSource)
\r
467 ts->tokenSource = tokenSource;
\r
470 static pANTLR3_STRING
\r
471 toString (pANTLR3_TOKEN_STREAM ts)
\r
473 pANTLR3_COMMON_TOKEN_STREAM cts;
\r
475 cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
\r
482 return ts->toStringSS(ts, 0, ts->istream->size(ts->istream));
\r
485 static pANTLR3_STRING
\r
486 toStringSS(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
\r
488 pANTLR3_STRING string;
\r
489 pANTLR3_TOKEN_SOURCE tsource;
\r
490 pANTLR3_COMMON_TOKEN tok;
\r
492 pANTLR3_COMMON_TOKEN_STREAM cts;
\r
494 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
\r
500 if (stop >= ts->istream->size(ts->istream))
\r
502 stop = ts->istream->size(ts->istream) - 1;
\r
505 /* Who is giving us these tokens?
\r
507 tsource = ts->getTokenSource(ts);
\r
509 if (tsource != NULL && cts->tokens != NULL)
\r
511 /* Finally, let's get a string
\r
513 string = tsource->strFactory->newRaw(tsource->strFactory);
\r
515 for (i = start; i <= stop; i++)
\r
517 tok = ts->get(ts, i);
\r
520 string->appendS(string, tok->getText(tok));
\r
530 static pANTLR3_STRING
\r
531 toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop)
\r
533 if (start != NULL && stop != NULL)
\r
535 return ts->toStringSS(ts, (ANTLR3_UINT32)start->getTokenIndex(start), (ANTLR3_UINT32)stop->getTokenIndex(stop));
\r
543 /** Move the input pointer to the next incoming token. The stream
\r
544 * must become active with LT(1) available. consume() simply
\r
545 * moves the input pointer so that LT(1) points at the next
\r
546 * input symbol. Consume at least one token.
\r
548 * Walk past any token not on the channel the parser is listening to.
\r
551 consume (pANTLR3_INT_STREAM is)
\r
553 pANTLR3_COMMON_TOKEN_STREAM cts;
\r
554 pANTLR3_TOKEN_STREAM ts;
\r
556 ts = (pANTLR3_TOKEN_STREAM) is->super;
\r
557 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
\r
559 if ((ANTLR3_UINT32)cts->p < cts->tokens->size(cts->tokens))
\r
562 cts->p = skipOffTokenChannels(cts, cts->p);
\r
567 /// As per ordinary consume but notifies the debugger about hidden
\r
568 /// tokens and so on.
\r
571 dbgConsume (pANTLR3_INT_STREAM is)
\r
573 pANTLR3_TOKEN_STREAM ts;
\r
576 pANTLR3_COMMON_TOKEN t;
\r
578 ts = (pANTLR3_TOKEN_STREAM) is->super;
\r
580 if (ts->initialStreamState == ANTLR3_TRUE)
\r
582 consumeInitialHiddenTokens(is);
\r
585 a = is->index(is); // Where are we right now?
\r
586 t = ts->_LT(ts, 1); // Current token from stream
\r
588 consume(is); // Standard consumer
\r
590 b = is->index(is); // Where are we after consuming 1 on channel token?
\r
592 ts->debugger->consumeToken(ts->debugger, t); // Tell the debugger that we consumed the first token
\r
596 // The standard consume caused the index to advance by more than 1,
\r
597 // which can only happen if it skipped some off-channel tokens.
\r
598 // we need to tell the debugger about those tokens.
\r
602 for (i = a+1; i<b; i++)
\r
604 ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, (ANTLR3_UINT32)i));
\r
610 /** A simple filter mechanism whereby you can tell this token stream
\r
611 * to force all tokens of type ttype to be on channel. For example,
\r
612 * when interpreting, we cannot execute actions so we need to tell
\r
613 * the stream to force all WS and NEWLINE to be a different, ignored,
\r
617 setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel)
\r
619 if (tokenStream->channelOverrides == NULL)
\r
621 tokenStream->channelOverrides = antlr3ListNew(10);
\r
624 /* We add one to the channel so we can distinguish NULL as being no entry in the
\r
625 * table for a particular token type.
\r
627 tokenStream->channelOverrides->put(tokenStream->channelOverrides, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)channel + 1), NULL);
\r
631 discardTokenType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 ttype)
\r
633 if (tokenStream->discardSet == NULL)
\r
635 tokenStream->discardSet = antlr3ListNew(31);
\r
638 /* We add one to the channel so we can distinguish NULL as being no entry in the
\r
639 * table for a particular token type. We could use bitsets for this I suppose too.
\r
641 tokenStream->discardSet->put(tokenStream->discardSet, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)ttype + 1), NULL);
\r
645 discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_BOOLEAN discard)
\r
647 tokenStream->discardOffChannel = discard;
\r
650 static pANTLR3_VECTOR
\r
651 getTokens (pANTLR3_COMMON_TOKEN_STREAM tokenStream)
\r
653 if (tokenStream->p == -1)
\r
655 fillBuffer(tokenStream);
\r
658 return tokenStream->tokens;
\r
661 static pANTLR3_LIST
\r
662 getTokenRange (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
\r
664 return tokenStream->getTokensSet(tokenStream, start, stop, NULL);
\r
666 /** Given a start and stop index, return a List of all tokens in
\r
667 * the token type BitSet. Return null if no tokens were found. This
\r
668 * method looks at both on and off channel tokens.
\r
670 static pANTLR3_LIST
\r
671 getTokensSet (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types)
\r
673 pANTLR3_LIST filteredList;
\r
676 pANTLR3_COMMON_TOKEN tok;
\r
678 if (tokenStream->p == -1)
\r
680 fillBuffer(tokenStream);
\r
682 if (stop > tokenStream->tstream->istream->size(tokenStream->tstream->istream))
\r
684 stop = tokenStream->tstream->istream->size(tokenStream->tstream->istream);
\r
691 /* We have the range set, now we need to iterate through the
\r
692 * installed tokens and create a new list with just the ones we want
\r
693 * in it. We are just moving pointers about really.
\r
695 filteredList = antlr3ListNew((ANTLR3_UINT32)tokenStream->tstream->istream->size(tokenStream->tstream->istream));
\r
697 for (i = start, n = 0; i<= stop; i++)
\r
699 tok = tokenStream->tstream->get(tokenStream->tstream, i);
\r
702 || types->isMember(types, tok->getType(tok) == ANTLR3_TRUE)
\r
705 filteredList->put(filteredList, n++, (void *)tok, NULL);
\r
709 /* Did we get any then?
\r
711 if (filteredList->size(filteredList) == 0)
\r
713 filteredList->free(filteredList);
\r
714 filteredList = NULL;
\r
717 return filteredList;
\r
720 static pANTLR3_LIST
\r
721 getTokensList (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list)
\r
723 pANTLR3_BITSET bitSet;
\r
724 pANTLR3_LIST newlist;
\r
726 bitSet = antlr3BitsetList(list->table);
\r
728 newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);
\r
730 bitSet->free(bitSet);
\r
736 static pANTLR3_LIST
\r
737 getTokensType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type)
\r
739 pANTLR3_BITSET bitSet;
\r
740 pANTLR3_LIST newlist;
\r
742 bitSet = antlr3BitsetOf(type, -1);
\r
743 newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);
\r
745 bitSet->free(bitSet);
\r
750 static ANTLR3_UINT32
\r
751 _LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
\r
753 pANTLR3_TOKEN_STREAM ts;
\r
754 pANTLR3_COMMON_TOKEN tok;
\r
756 ts = (pANTLR3_TOKEN_STREAM) is->super;
\r
758 tok = ts->_LT(ts, i);
\r
762 return tok->getType(tok);
\r
766 return ANTLR3_TOKEN_INVALID;
\r
770 /// As per _LA() but for debug mode.
\r
772 static ANTLR3_UINT32
\r
773 dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
\r
775 pANTLR3_TOKEN_STREAM ts;
\r
777 ts = (pANTLR3_TOKEN_STREAM) is->super;
\r
779 if (ts->initialStreamState == ANTLR3_TRUE)
\r
781 consumeInitialHiddenTokens(is);
\r
783 ts->debugger->LT(ts->debugger, i, tokLT(ts, i));
\r
787 static ANTLR3_MARKER
\r
788 mark (pANTLR3_INT_STREAM is)
\r
790 is->lastMarker = is->index(is);
\r
791 return is->lastMarker;
\r
794 /// As per mark() but with a call to tell the debugger we are doing this
\r
796 static ANTLR3_MARKER
\r
797 dbgMark (pANTLR3_INT_STREAM is)
\r
799 pANTLR3_TOKEN_STREAM ts;
\r
801 ts = (pANTLR3_TOKEN_STREAM) is->super;
\r
803 is->lastMarker = is->index(is);
\r
804 ts->debugger->mark(ts->debugger, is->lastMarker);
\r
806 return is->lastMarker;
\r
810 release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark)
\r
815 static ANTLR3_UINT32
\r
816 size (pANTLR3_INT_STREAM is)
\r
818 pANTLR3_COMMON_TOKEN_STREAM cts;
\r
819 pANTLR3_TOKEN_STREAM ts;
\r
821 if (is->cachedSize > 0)
\r
823 return is->cachedSize;
\r
825 ts = (pANTLR3_TOKEN_STREAM) is->super;
\r
826 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
\r
828 is->cachedSize = cts->tokens->count;
\r
829 return is->cachedSize;
\r
832 static ANTLR3_MARKER
\r
833 tindex (pANTLR3_INT_STREAM is)
\r
835 pANTLR3_COMMON_TOKEN_STREAM cts;
\r
836 pANTLR3_TOKEN_STREAM ts;
\r
838 ts = (pANTLR3_TOKEN_STREAM) is->super;
\r
839 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
\r
845 dbgRewindLast (pANTLR3_INT_STREAM is)
\r
847 pANTLR3_TOKEN_STREAM ts;
\r
849 ts = (pANTLR3_TOKEN_STREAM) is->super;
\r
851 ts->debugger->rewindLast(ts->debugger);
\r
853 is->rewind(is, is->lastMarker);
\r
856 rewindLast (pANTLR3_INT_STREAM is)
\r
858 is->rewind(is, is->lastMarker);
\r
861 rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
\r
863 is->seek(is, (ANTLR3_UINT32)(marker));
\r
866 dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
\r
868 pANTLR3_TOKEN_STREAM ts;
\r
870 ts = (pANTLR3_TOKEN_STREAM) is->super;
\r
872 ts->debugger->rewind(ts->debugger, marker);
\r
874 is->seek(is, (ANTLR3_UINT32)(marker));
\r
878 seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
\r
880 pANTLR3_COMMON_TOKEN_STREAM cts;
\r
881 pANTLR3_TOKEN_STREAM ts;
\r
883 ts = (pANTLR3_TOKEN_STREAM) is->super;
\r
884 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
\r
886 cts->p = (ANTLR3_UINT32)index;
\r
889 dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
\r
891 // TODO: Implement seek in debugger when Ter adds it to Java
\r
896 fillBufferExt(pANTLR3_COMMON_TOKEN_STREAM tokenStream)
\r
898 fillBuffer(tokenStream);
\r
901 fillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream) {
\r
902 ANTLR3_UINT32 index;
\r
903 pANTLR3_COMMON_TOKEN tok;
\r
904 ANTLR3_BOOLEAN discard;
\r
907 /* Start at index 0 of course
\r
911 /* Pick out the next token from the token source
\r
912 * Remember we just get a pointer (reference if you like) here
\r
913 * and so if we store it anywhere, we don't set any pointers to auto free it.
\r
915 tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);
\r
917 while (tok != NULL && tok->type != ANTLR3_TOKEN_EOF)
\r
919 discard = ANTLR3_FALSE; /* Assume we are not discarding */
\r
921 /* I employ a bit of a trick, or perhaps hack here. Rather than
\r
922 * store a pointer to a structure in the override map and discard set
\r
923 * we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0
\r
924 * we can distinguish "not being there" from "being channel or type 0"
\r
927 if (tokenStream->discardSet != NULL
\r
928 && tokenStream->discardSet->get(tokenStream->discardSet, tok->getType(tok)) != NULL)
\r
930 discard = ANTLR3_TRUE;
\r
932 else if ( tokenStream->discardOffChannel == ANTLR3_TRUE
\r
933 && tok->getChannel(tok) != tokenStream->channel
\r
936 discard = ANTLR3_TRUE;
\r
938 else if (tokenStream->channelOverrides != NULL)
\r
940 /* See if this type is in the override map
\r
942 channelI = tokenStream->channelOverrides->get(tokenStream->channelOverrides, tok->getType(tok) + 1);
\r
944 if (channelI != NULL)
\r
948 tok->setChannel(tok, ANTLR3_UINT32_CAST(channelI) - 1);
\r
952 /* If not discarding it, add it to the list at the current index
\r
954 if (discard == ANTLR3_FALSE)
\r
956 /* Add it, indicating that we will delete it and the table should not
\r
958 tok->setTokenIndex(tok, index);
\r
960 tokenStream->tokens->add(tokenStream->tokens, (void *) tok, NULL);
\r
964 tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);
\r
967 /* Cache the size so we don't keep doing indirect method calls. We do this as
\r
968 * early as possible so that anything after this may utilize the cached value.
\r
970 tokenStream->tstream->istream->cachedSize = tokenStream->tokens->count;
\r
972 /* Set the consume pointer to the first token that is on our channel
\r
974 tokenStream->p = 0;
\r
975 tokenStream->p = skipOffTokenChannels(tokenStream, tokenStream->p);
\r
979 /// Given a starting index, return the index of the first on-channel
\r
982 static ANTLR3_UINT32
\r
983 skipOffTokenChannels(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i) {
\r
985 pANTLR3_COMMON_TOKEN tok;
\r
987 n = tokenStream->tstream->istream->cachedSize;
\r
991 tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[i].element;
\r
993 if (tok->channel!= tokenStream->channel)
\r
1005 static ANTLR3_UINT32
\r
1006 skipOffTokenChannelsReverse(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 x)
\r
1008 pANTLR3_COMMON_TOKEN tok;
\r
1012 tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[x].element;
\r
1014 if ((tok->channel != tokenStream->channel))
\r
1026 /// Return a string that represents the name assoicated with the input source
\r
1028 /// /param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream.
\r
1031 /// /implements ANTLR3_INT_STREAM_struct::getSourceName()
\r
1033 static pANTLR3_STRING
\r
1034 getSourceName (pANTLR3_INT_STREAM is)
\r
1036 // Slightly convoluted as we must trace back to the lexer's input source
\r
1037 // via the token source. The streamName that is here is not initialized
\r
1038 // because this is a token stream, not a file or string stream, which are the
\r
1039 // only things that have a context for a source name.
\r
1041 return ((pANTLR3_TOKEN_STREAM)(is->super))->tokenSource->fileName;
\r