bundles/org.simantics.databoard/cpp/DataBoardTest/libantlr3c-3.2/src/antlr3commontoken.c

   1 /**
   2  * Contains the default implementation of the common token used within
   3  * java. Custom tokens should create this structure and then append to it using the
   4  * custom pointer to install their own structure and API.
   5  */
   6
   7 // [The "BSD licence"]
   8 // Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
   9 // http://www.temporal-wave.com
  10 // http://www.linkedin.com/in/jimidle
  11 //
  12 // All rights reserved.
  13 //
  14 // Redistribution and use in source and binary forms, with or without
  15 // modification, are permitted provided that the following conditions
  16 // are met:
  17 // 1. Redistributions of source code must retain the above copyright
  18 //    notice, this list of conditions and the following disclaimer.
  19 // 2. Redistributions in binary form must reproduce the above copyright
  20 //    notice, this list of conditions and the following disclaimer in the
  21 //    documentation and/or other materials provided with the distribution.
  22 // 3. The name of the author may not be used to endorse or promote products
  23 //    derived from this software without specific prior written permission.
  24 //
  25 // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  26 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  27 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  28 // IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  29 // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  30 // NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  31 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  32 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  33 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  34 // THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35
  36 #include    <antlr3.h>
  37
  38 /* Token API
  39  */
  40 static  pANTLR3_STRING  getText                                 (pANTLR3_COMMON_TOKEN token);
  41 static  void                    setText                                 (pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text);
  42 static  void                    setText8                                (pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text);
  43 static  ANTLR3_UINT32   getType                                 (pANTLR3_COMMON_TOKEN token);
  44 static  void                    setType                                 (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type);
  45 static  ANTLR3_UINT32   getLine                                 (pANTLR3_COMMON_TOKEN token);
  46 static  void                    setLine                                 (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line);
  47 static  ANTLR3_INT32    getCharPositionInLine   (pANTLR3_COMMON_TOKEN token);
  48 static  void                    setCharPositionInLine   (pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos);
  49 static  ANTLR3_UINT32   getChannel                              (pANTLR3_COMMON_TOKEN token);
  50 static  void                    setChannel                              (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel);
  51 static  ANTLR3_MARKER   getTokenIndex                   (pANTLR3_COMMON_TOKEN token);
  52 static  void                    setTokenIndex                   (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER);
  53 static  ANTLR3_MARKER   getStartIndex                   (pANTLR3_COMMON_TOKEN token);
  54 static  void                    setStartIndex                   (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);
  55 static  ANTLR3_MARKER   getStopIndex                    (pANTLR3_COMMON_TOKEN token);
  56 static  void                    setStopIndex                    (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);
  57 static  pANTLR3_STRING  toString                                (pANTLR3_COMMON_TOKEN token);
  58
  59 /* Factory API
  60  */
  61 static  void                    factoryClose    (pANTLR3_TOKEN_FACTORY factory);
  62 static  pANTLR3_COMMON_TOKEN    newToken        (void);
  63 static  void                    setInputStream  (pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input);
  64
  65 /* Internal management functions
  66  */
  67 static  void                    newPool         (pANTLR3_TOKEN_FACTORY factory);
  68 static  pANTLR3_COMMON_TOKEN    newPoolToken    (pANTLR3_TOKEN_FACTORY factory);
  69
  70
  71
  72 ANTLR3_API pANTLR3_COMMON_TOKEN
  73 antlr3CommonTokenNew(ANTLR3_UINT32 ttype)
  74 {
  75         pANTLR3_COMMON_TOKEN    token;
  76
  77         // Create a raw token with the interface installed
  78         //
  79         token   = newToken();
  80
  81         if      (token != NULL)
  82         {
  83                 token->setType(token, ttype);
  84         }
  85
  86         // All good
  87         //
  88         return  token;
  89 }
  90
  91 ANTLR3_API pANTLR3_TOKEN_FACTORY
  92 antlr3TokenFactoryNew(pANTLR3_INPUT_STREAM input)
  93 {
  94     pANTLR3_TOKEN_FACTORY   factory;
  95
  96     /* allocate memory
  97      */
  98     factory     = (pANTLR3_TOKEN_FACTORY) ANTLR3_MALLOC((size_t)sizeof(ANTLR3_TOKEN_FACTORY));
  99
 100     if  (factory == NULL)
 101     {
 102         return  NULL;
 103     }
 104
 105     /* Install factory API
 106      */
 107     factory->newToken       =  newPoolToken;
 108     factory->close                      =  factoryClose;
 109     factory->setInputStream = setInputStream;
 110
 111     /* Allocate the initial pool
 112      */
 113     factory->thisPool   = -1;
 114     factory->pools      = NULL;
 115     newPool(factory);
 116
 117     /* Factory space is good, we now want to initialize our cheating token
 118      * which one it is initialized is the model for all tokens we manufacture
 119      */
 120     antlr3SetTokenAPI(&factory->unTruc);
 121
 122     /* Set some initial variables for future copying
 123      */
 124     factory->unTruc.factoryMade = ANTLR3_TRUE;
 125
 126     // Input stream
 127     //
 128     setInputStream(factory, input);
 129
 130     return  factory;
 131
 132 }
 133
 134 static void
 135 setInputStream  (pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input)
 136 {
 137     factory->input          =  input;
 138     factory->unTruc.input   =  input;
 139         if      (input != NULL)
 140         {
 141                 factory->unTruc.strFactory      = input->strFactory;
 142         }
 143         else
 144         {
 145                 factory->unTruc.strFactory = NULL;
 146     }
 147 }
 148
 149 static void
 150 newPool(pANTLR3_TOKEN_FACTORY factory)
 151 {
 152     /* Increment factory count
 153      */
 154     factory->thisPool++;
 155
 156     /* Ensure we have enough pointers allocated
 157      */
 158     factory->pools = (pANTLR3_COMMON_TOKEN *)
 159                      ANTLR3_REALLOC(    (void *)factory->pools,     /* Current pools pointer (starts at NULL)   */
 160                                         (ANTLR3_UINT32)((factory->thisPool + 1) * sizeof(pANTLR3_COMMON_TOKEN *))       /* Memory for new pool pointers */
 161                                         );
 162
 163     /* Allocate a new pool for the factory
 164      */
 165     factory->pools[factory->thisPool]   =
 166                             (pANTLR3_COMMON_TOKEN)
 167                                 ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_COMMON_TOKEN) * ANTLR3_FACTORY_POOL_SIZE));
 168
 169     /* Reset the counters
 170      */
 171     factory->nextToken  = 0;
 172
 173     /* Done
 174      */
 175     return;
 176 }
 177
 178 static pANTLR3_COMMON_TOKEN
 179 newPoolToken(pANTLR3_TOKEN_FACTORY factory)
 180 {
 181     pANTLR3_COMMON_TOKEN token;
 182
 183     /* See if we need a new token pool before allocating a new
 184      * one
 185      */
 186     if (factory->nextToken >= ANTLR3_FACTORY_POOL_SIZE)
 187     {
 188         /* We ran out of tokens in the current pool, so we need a new pool
 189          */
 190         newPool(factory);
 191     }
 192
 193     /* Assuming everything went well (we are trying for performance here so doing minimal
 194      * error checking. Then we can work out what the pointer is to the next token.
 195      */
 196     token = factory->pools[factory->thisPool] + factory->nextToken;
 197     factory->nextToken++;
 198
 199     /* We have our token pointer now, so we can initialize it to the predefined model.
 200      */
 201     antlr3SetTokenAPI(token);
 202
 203     /* It is factory made, and we need to copy the string factory pointer
 204      */
 205     token->factoryMade  = ANTLR3_TRUE;
 206     token->strFactory   = factory->input == NULL ? NULL : factory->input->strFactory;
 207     token->input        = factory->input;
 208
 209     /* And we are done
 210      */
 211     return token;
 212 }
 213
 214 static  void
 215 factoryClose        (pANTLR3_TOKEN_FACTORY factory)
 216 {
 217     pANTLR3_COMMON_TOKEN    pool;
 218     ANTLR3_INT32            poolCount;
 219     ANTLR3_UINT32           limit;
 220     ANTLR3_UINT32           token;
 221     pANTLR3_COMMON_TOKEN    check;
 222
 223     /* We iterate the token pools one at a time
 224      */
 225     for (poolCount = 0; poolCount <= factory->thisPool; poolCount++)
 226     {
 227         /* Pointer to current pool
 228          */
 229         pool    = factory->pools[poolCount];
 230
 231         /* Work out how many tokens we need to check in this pool.
 232          */
 233         limit   = (poolCount == factory->thisPool ? factory->nextToken : ANTLR3_FACTORY_POOL_SIZE);
 234
 235         /* Marginal condition, we might be at the start of a brand new pool
 236          * where the nextToken is 0 and nothing has been allocated.
 237          */
 238         if  (limit > 0)
 239         {
 240             /* We have some tokens allocated from this pool
 241              */
 242             for (token = 0; token < limit; token++)
 243             {
 244                 /* Next one in the chain
 245                  */
 246                 check   = pool + token;
 247
 248                 /* If the programmer made this a custom token, then
 249                  * see if we need to call their free routine.
 250                  */
 251                 if  (check->custom != NULL && check->freeCustom != NULL)
 252                 {
 253                     check->freeCustom(check->custom);
 254                     check->custom = NULL;
 255                 }
 256             }
 257         }
 258
 259         /* We can now free this pool allocation
 260          */
 261         ANTLR3_FREE(factory->pools[poolCount]);
 262         factory->pools[poolCount] = NULL;
 263     }
 264
 265     /* All the pools are deallocated we can free the pointers to the pools
 266      * now.
 267      */
 268     ANTLR3_FREE(factory->pools);
 269
 270     /* Finally, we can free the space for the factory itself
 271      */
 272     ANTLR3_FREE(factory);
 273 }
 274
 275
 276 static  pANTLR3_COMMON_TOKEN
 277 newToken(void)
 278 {
 279     pANTLR3_COMMON_TOKEN    token;
 280
 281     /* Allocate memory for this
 282      */
 283     token   = (pANTLR3_COMMON_TOKEN) ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_COMMON_TOKEN)));
 284
 285     if  (token == NULL)
 286     {
 287         return  NULL;
 288     }
 289
 290     // Install the API
 291     //
 292     antlr3SetTokenAPI(token);
 293     token->factoryMade = ANTLR3_FALSE;
 294
 295     return  token;
 296 }
 297
 298 ANTLR3_API void
 299 antlr3SetTokenAPI(pANTLR3_COMMON_TOKEN token)
 300 {
 301     token->getText                                      = getText;
 302     token->setText                                      = setText;
 303     token->setText8                                     = setText8;
 304     token->getType                                      = getType;
 305     token->setType                                      = setType;
 306     token->getLine                                      = getLine;
 307     token->setLine                                  = setLine;
 308     token->setLine                                      = setLine;
 309     token->getCharPositionInLine    = getCharPositionInLine;
 310     token->setCharPositionInLine    = setCharPositionInLine;
 311     token->getChannel                           = getChannel;
 312     token->setChannel                           = setChannel;
 313     token->getTokenIndex                        = getTokenIndex;
 314     token->setTokenIndex                        = setTokenIndex;
 315     token->getStartIndex                        = getStartIndex;
 316     token->setStartIndex                        = setStartIndex;
 317     token->getStopIndex                         = getStopIndex;
 318     token->setStopIndex                         = setStopIndex;
 319     token->toString                                     = toString;
 320
 321     // Set defaults
 322     //
 323     token->setCharPositionInLine(token, -1);
 324
 325     token->custom                   = NULL;
 326     token->freeCustom       = NULL;
 327     token->type                     = ANTLR3_TOKEN_INVALID;
 328         token->textState                = ANTLR3_TEXT_NONE;
 329     token->start                    = 0;
 330     token->stop                     = 0;
 331     token->channel                  = ANTLR3_TOKEN_DEFAULT_CHANNEL;
 332     token->line                     = 0;
 333     token->index                    = 0;
 334     token->input                    = NULL;
 335         token->user1                    = 0;
 336         token->user2                    = 0;
 337         token->user3                    = 0;
 338         token->custom                   = NULL;
 339
 340     return;
 341 }
 342
 343 static  pANTLR3_STRING  getText                 (pANTLR3_COMMON_TOKEN token)
 344 {
 345         switch (token->textState)
 346         {
 347                 case ANTLR3_TEXT_STRING:
 348
 349                         // Someone already created a string for this token, so we just
 350                         // use it.
 351                         //
 352                         return  token->tokText.text;
 353                         break;
 354
 355                 case ANTLR3_TEXT_CHARP:
 356
 357                         // We had a straight text pointer installed, now we
 358                         // must convert it to a string. Note we have to do this here
 359                         // or otherwise setText8() will just install the same char*
 360                         //
 361                         if      (token->strFactory != NULL)
 362                         {
 363                                 token->tokText.text     = token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)token->tokText.chars);
 364                                 token->textState        = ANTLR3_TEXT_STRING;
 365                                 return token->tokText.text;
 366                         }
 367                         else
 368                         {
 369                                 // We cannot do anything here
 370                                 //
 371                                 return NULL;
 372                         }
 373                         break;
 374
 375                 default:
 376
 377                         // EOF is a special case
 378                         //
 379                         if (token->type == ANTLR3_TOKEN_EOF)
 380                         {
 381                                 token->tokText.text     = token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)"<EOF>");
 382                                 token->textState        = ANTLR3_TEXT_STRING;
 383                                 return token->tokText.text;
 384                         }
 385
 386
 387                         // We had nothing installed in the token, create a new string
 388                         // from the input stream
 389                         //
 390
 391                         if      (token->input != NULL)
 392                         {
 393
 394                                 return  token->input->substr(   token->input,
 395                                                                                                 token->getStartIndex(token),
 396                                                                                                 token->getStopIndex(token)
 397                                                                                         );
 398                         }
 399
 400                         // Nothing to return, there is no input stream
 401                         //
 402                         return NULL;
 403                         break;
 404         }
 405 }
 406 static  void            setText8                (pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text)
 407 {
 408         // No text to set, so ignore
 409         //
 410         if      (text == NULL) return;
 411
 412         switch  (token->textState)
 413         {
 414                 case    ANTLR3_TEXT_NONE:
 415                 case    ANTLR3_TEXT_CHARP:      // Caller must free before setting again, if it needs to be freed
 416
 417                         // Nothing in there yet, or just a char *, so just set the
 418                         // text as a pointer
 419                         //
 420                         token->textState                = ANTLR3_TEXT_CHARP;
 421                         token->tokText.chars    = (pANTLR3_UCHAR)text;
 422                         break;
 423
 424                 default:
 425
 426                         // It was already a pANTLR3_STRING, so just override it
 427                         //
 428                         token->tokText.text->set8(token->tokText.text, (const char *)text);
 429                         break;
 430         }
 431
 432         // We are done
 433         //
 434         return;
 435 }
 436
 437 /** \brief Install the supplied text string as teh text for the token.
 438  * The method assumes that the existing text (if any) was created by a factory
 439  * and so does not attempt to release any memory it is using.Text not created
 440  * by a string fctory (not advised) should be released prior to this call.
 441  */
 442 static  void            setText                 (pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text)
 443 {
 444         // Merely replaces and existing pre-defined text with the supplied
 445         // string
 446         //
 447         token->textState        = ANTLR3_TEXT_STRING;
 448         token->tokText.text     = text;
 449
 450         /* We are done
 451         */
 452         return;
 453 }
 454
 455 static  ANTLR3_UINT32   getType                 (pANTLR3_COMMON_TOKEN token)
 456 {
 457     return  token->type;
 458 }
 459
 460 static  void            setType                 (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type)
 461 {
 462     token->type = type;
 463 }
 464
 465 static  ANTLR3_UINT32   getLine                 (pANTLR3_COMMON_TOKEN token)
 466 {
 467     return  token->line;
 468 }
 469
 470 static  void            setLine                 (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line)
 471 {
 472     token->line = line;
 473 }
 474
 475 static  ANTLR3_INT32    getCharPositionInLine   (pANTLR3_COMMON_TOKEN token)
 476 {
 477     return  token->charPosition;
 478 }
 479
 480 static  void            setCharPositionInLine   (pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos)
 481 {
 482     token->charPosition = pos;
 483 }
 484
 485 static  ANTLR3_UINT32   getChannel              (pANTLR3_COMMON_TOKEN token)
 486 {
 487     return  token->channel;
 488 }
 489
 490 static  void            setChannel              (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel)
 491 {
 492     token->channel  = channel;
 493 }
 494
 495 static  ANTLR3_MARKER   getTokenIndex           (pANTLR3_COMMON_TOKEN token)
 496 {
 497     return  token->index;
 498 }
 499
 500 static  void            setTokenIndex           (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index)
 501 {
 502     token->index    = index;
 503 }
 504
 505 static  ANTLR3_MARKER   getStartIndex           (pANTLR3_COMMON_TOKEN token)
 506 {
 507         return  token->start == -1 ? (ANTLR3_MARKER)(token->input->data) : token->start;
 508 }
 509
 510 static  void            setStartIndex           (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER start)
 511 {
 512     token->start    = start;
 513 }
 514
 515 static  ANTLR3_MARKER   getStopIndex            (pANTLR3_COMMON_TOKEN token)
 516 {
 517     return  token->stop;
 518 }
 519
 520 static  void            setStopIndex            (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER stop)
 521 {
 522     token->stop = stop;
 523 }
 524
 525 static  pANTLR3_STRING    toString              (pANTLR3_COMMON_TOKEN token)
 526 {
 527     pANTLR3_STRING  text;
 528     pANTLR3_STRING  outtext;
 529
 530     text    =   token->getText(token);
 531
 532     if  (text == NULL)
 533     {
 534                 return NULL;
 535     }
 536
 537         if      (text->factory == NULL)
 538         {
 539                 return text;            // This usall ymeans it is the EOF token
 540         }
 541
 542     /* A new empty string to assemble all the stuff in
 543      */
 544     outtext = text->factory->newRaw(text->factory);
 545
 546     /* Now we use our handy dandy string utility to assemble the
 547      * the reporting string
 548      * return "[@"+getTokenIndex()+","+start+":"+stop+"='"+txt+"',<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+"]";
 549      */
 550     outtext->append8(outtext, "[Index: ");
 551     outtext->addi   (outtext, (ANTLR3_INT32)token->getTokenIndex(token));
 552     outtext->append8(outtext, " (Start: ");
 553     outtext->addi   (outtext, (ANTLR3_INT32)token->getStartIndex(token));
 554     outtext->append8(outtext, "-Stop: ");
 555     outtext->addi   (outtext, (ANTLR3_INT32)token->getStopIndex(token));
 556     outtext->append8(outtext, ") ='");
 557     outtext->appendS(outtext, text);
 558     outtext->append8(outtext, "', type<");
 559     outtext->addi   (outtext, token->type);
 560     outtext->append8(outtext, "> ");
 561
 562     if  (token->getChannel(token) > ANTLR3_TOKEN_DEFAULT_CHANNEL)
 563     {
 564         outtext->append8(outtext, "(channel = ");
 565         outtext->addi   (outtext, (ANTLR3_INT32)token->getChannel(token));
 566         outtext->append8(outtext, ") ");
 567     }
 568
 569     outtext->append8(outtext, "Line: ");
 570     outtext->addi   (outtext, (ANTLR3_INT32)token->getLine(token));
 571     outtext->append8(outtext, " LinePos:");
 572     outtext->addi   (outtext, token->getCharPositionInLine(token));
 573     outtext->addc   (outtext, ']');
 574
 575     return  outtext;
 576 }
 577