Tools/CCode/Source/Pccts/h/ATokenBuffer.cpp

   1 /* ANTLRTokenBuffer.cpp
   2  *
   3  * SOFTWARE RIGHTS
   4  *
   5  * We reserve no LEGAL rights to the Purdue Compiler Construction Tool
   6  * Set (PCCTS) -- PCCTS is in the public domain.  An individual or
   7  * company may do whatever they wish with source code distributed with
   8  * PCCTS or the code generated by PCCTS, including the incorporation of
   9  * PCCTS, or its output, into commerical software.
  10  *
  11  * We encourage users to develop software with PCCTS.  However, we do ask
  12  * that credit is given to us for developing PCCTS.  By "credit",
  13  * we mean that if you incorporate our source code into one of your
  14  * programs (commercial product, research project, or otherwise) that you
  15  * acknowledge this fact somewhere in the documentation, research report,
  16  * etc...  If you like PCCTS and have developed a nice tool with the
  17  * output, please mention that you developed it using PCCTS.  In
  18  * addition, we ask that this header remain intact in our source code.
  19  * As long as these guidelines are kept, we expect to continue enhancing
  20  * this system and expect to make other tools available as they are
  21  * completed.
  22  *
  23  * ANTLR 1.33
  24  * Terence Parr
  25  * Parr Research Corporation
  26  * with Purdue University and AHPCRC, University of Minnesota
  27  * 1989-2000
  28  */
  29
  30 typedef int ANTLRTokenType;     // fool AToken.h into compiling
  31
  32 class ANTLRParser;                                      /* MR1 */
  33
  34 #define ANTLR_SUPPORT_CODE
  35
  36 #include "pcctscfg.h"
  37
  38 #include ATOKENBUFFER_H
  39 #include APARSER_H              // MR23
  40
  41 typedef ANTLRAbstractToken *_ANTLRTokenPtr;
  42
  43 #if defined(DBG_TBUF)||defined(DBG_TBUF_MARK_REW)
  44 static unsigned char test[1000];
  45 #endif
  46
  47 #ifdef DBG_REFCOUNTTOKEN
  48 int ANTLRRefCountToken::ctor = 0; /* MR23 */
  49 int ANTLRRefCountToken::dtor = 0; /* MR23 */
  50 #endif
  51
  52 ANTLRTokenBuffer::
  53 ANTLRTokenBuffer(ANTLRTokenStream *_input, int _k, int _chunk_size_formal) /* MR14 */
  54 {
  55         this->input = _input;
  56         this->k = _k;
  57         buffer_size = chunk_size = _chunk_size_formal;
  58         buffer = (_ANTLRTokenPtr *)
  59                          calloc(chunk_size+1,sizeof(_ANTLRTokenPtr ));
  60         if ( buffer == NULL ) {
  61                 panic("cannot alloc token buffer");
  62         }
  63         buffer++;                               // leave the first elem empty so tp-1 is valid ptr
  64
  65         tp = &buffer[0];
  66         last = tp-1;
  67         next = &buffer[0];
  68         num_markers = 0;
  69         end_of_buffer = &buffer[buffer_size-1];
  70         threshold = &buffer[(int)(buffer_size/2)];      // MR23 - Used to be 1.0/2.0 !
  71         _deleteTokens = 1;      // assume we delete tokens
  72         parser=NULL;                            // MR5 - uninitialized reference
  73 }
  74
  75 static void f() {;}
  76 ANTLRTokenBuffer::
  77 ~ANTLRTokenBuffer()
  78 {
  79         f();
  80         // Delete all remaining tokens (from 0..last inclusive)
  81         if ( _deleteTokens )
  82         {
  83                 _ANTLRTokenPtr *z;
  84                 for (z=buffer; z<=last; z++)
  85                 {
  86                         (*z)->deref();
  87 //                      z->deref();
  88 #ifdef DBG_REFCOUNTTOKEN
  89                                         /* MR23 */ printMessage(stderr, "##########dtor: deleting token '%s' (ref %d)\n",
  90                                                         ((ANTLRCommonToken *)*z)->getText(), (*z)->nref());
  91 #endif
  92                         if ( (*z)->nref()==0 )
  93                         {
  94                                 delete (*z);
  95                         }
  96                 }
  97         }
  98
  99         if ( buffer!=NULL ) free((char *)(buffer-1));
 100 }
 101
 102 #if defined(DBG_TBUF)||defined(DBG_TBUF_MARK_REW)
 103 #include "pccts_stdio.h"
 104 PCCTS_NAMESPACE_STD
 105 #endif
 106
 107 _ANTLRTokenPtr ANTLRTokenBuffer::
 108 getToken()
 109 {
 110         if ( tp <= last )       // is there any buffered lookahead still to be read?
 111         {
 112                 return *tp++;   // read buffered lookahead
 113         }
 114         // out of buffered lookahead, get some more "real"
 115         // input from getANTLRToken()
 116         if ( num_markers==0 )
 117         {
 118                 if( next > threshold )
 119                 {
 120 #ifdef DBG_TBUF
 121 /* MR23 */ printMessage(stderr,"getToken: next > threshold (high water is %d)\n", threshold-buffer);
 122 #endif
 123                         makeRoom();
 124                 }
 125         }
 126         else {
 127                 if ( next > end_of_buffer )
 128                 {
 129 #ifdef DBG_TBUF
 130 /* MR23 */ printMessage(stderr,"getToken: next > end_of_buffer (size is %d)\n", buffer_size);
 131 #endif
 132                         extendBuffer();
 133                 }
 134         }
 135         *next = getANTLRToken();
 136         (*next)->ref();                         // say we have a copy of this pointer in buffer
 137         last = next;
 138         next++;
 139         tp = last;
 140         return *tp++;
 141 }
 142
 143 void ANTLRTokenBuffer::
 144 rewind(int pos)
 145 {
 146 #if defined(DBG_TBUF)||defined(DBG_TBUF_MARK_REW)
 147         /* MR23 */ printMessage(stderr, "rewind(%d)[nm=%d,from=%d,%d.n=%d]\n", pos, num_markers, tp-buffer,pos,test[pos]);
 148         test[pos]--;
 149 #endif
 150         tp = &buffer[pos];
 151         num_markers--;
 152 }
 153
 154 /*
 155  * This function is used to specify that the token pointers read
 156  * by the ANTLRTokenBuffer should be buffered up (to be reused later).
 157  */
 158 int ANTLRTokenBuffer::
 159 mark()
 160 {
 161 #if defined(DBG_TBUF)||defined(DBG_TBUF_MARK_REW)
 162         test[tp-buffer]++;
 163         /* MR23 */ printMessage(stderr,"mark(%d)[nm=%d,%d.n=%d]\n",tp-buffer,num_markers+1,tp-buffer,test[tp-buffer]);
 164 #endif
 165         num_markers++;
 166         return tp - buffer;
 167 }
 168
 169 /*
 170  * returns the token pointer n positions ahead.
 171  * This implies that bufferedToken(1) gets the NEXT symbol of lookahead.
 172  * This is used in conjunction with the ANTLRParser lookahead buffer.
 173  *
 174  * No markers are set or anything.  A bunch of input is buffered--that's all.
 175  * The tp pointer is left alone as the lookahead has not been advanced
 176  * with getToken().  The next call to getToken() will find a token
 177  * in the buffer and won't have to call getANTLRToken().
 178  *
 179  * If this is called before a consume() is done, how_many_more_i_need is
 180  * set to 'n'.
 181  */
 182 _ANTLRTokenPtr ANTLRTokenBuffer::
 183 bufferedToken(int n)
 184 {
 185 //      int how_many_more_i_need = (last-tp < 0) ? n : n-(last-tp)-1;
 186         int how_many_more_i_need = (tp > last) ? n : n-(last-tp)-1;
 187         // Make sure that at least n tokens are available in the buffer
 188 #ifdef DBG_TBUF
 189         /* MR23 */ printMessage(stderr, "bufferedToken(%d)\n", n);
 190 #endif
 191         for (int i=1; i<=how_many_more_i_need; i++)
 192         {
 193                 if ( next > end_of_buffer )     // buffer overflow?
 194                 {
 195                         extendBuffer();
 196                 }
 197                 *next = getANTLRToken();
 198                 (*next)->ref();         // say we have a copy of this pointer in buffer
 199                 last = next;
 200                 next++;
 201         }
 202         return tp[n - 1];
 203 }
 204
 205 /* If no markers are set, the none of the input needs to be saved (except
 206  * for the lookahead Token pointers).  We save only k-1 token pointers as
 207  * we are guaranteed to do a getANTLRToken() right after this because otherwise
 208  * we wouldn't have needed to extend the buffer.
 209  *
 210  * If there are markers in the buffer, we need to save things and so
 211  * extendBuffer() is called.
 212  */
 213 void ANTLRTokenBuffer::
 214 makeRoom()
 215 {
 216 #ifdef DBG_TBUF
 217         /* MR23 */ printMessage(stderr, "in makeRoom.................\n");
 218         /* MR23 */ printMessage(stderr, "num_markers==%d\n", num_markers);
 219 #endif
 220 /*
 221         if ( num_markers == 0 )
 222         {
 223 */
 224 #ifdef DBG_TBUF
 225                 /* MR23 */ printMessage(stderr, "moving lookahead and resetting next\n");
 226
 227                 _ANTLRTokenPtr *r;
 228                 /* MR23 */ printMessage(stderr, "tbuf = [");
 229                 for (r=buffer; r<=last; r++)
 230                 {
 231                         if ( *r==NULL ) /* MR23 */ printMessage(stderr, " xxx");
 232                         else /* MR23 */ printMessage(stderr, " '%s'", ((ANTLRCommonToken *)*r)->getText());
 233                 }
 234                 /* MR23 */ printMessage(stderr, " ]\n");
 235
 236                 /* MR23 */ printMessage(stderr,
 237                 "before: tp=%d, last=%d, next=%d, threshold=%d\n",tp-buffer,last-buffer,next-buffer,threshold-buffer);
 238 #endif
 239
 240                 // Delete all tokens from 0..last-(k-1) inclusive
 241                 if ( _deleteTokens )
 242                 {
 243                         _ANTLRTokenPtr *z;
 244                         for (z=buffer; z<=last-(k-1); z++)
 245                         {
 246                                 (*z)->deref();
 247 //                              z->deref();
 248 #ifdef DBG_REFCOUNTTOKEN
 249                                         /* MR23 */ printMessage(stderr, "##########makeRoom: deleting token '%s' (ref %d)\n",
 250                                                         ((ANTLRCommonToken *)*z)->getText(), (*z)->nref());
 251 #endif
 252                                 if ( (*z)->nref()==0 )
 253                                 {
 254                                         delete (*z);
 255                                 }
 256                         }
 257                 }
 258
 259                 // reset the buffer to initial conditions, but move k-1 symbols
 260                 // to the beginning of buffer and put new input symbol at k
 261                 _ANTLRTokenPtr *p = buffer, *q = last-(k-1)+1;
 262 //              ANTLRAbstractToken **p = buffer, **q = end_of_buffer-(k-1)+1;
 263 #ifdef DBG_TBUF
 264                 /* MR23 */ printMessage(stderr, "lookahead buffer = [");
 265 #endif
 266                 for (int i=1; i<=(k-1); i++)
 267                 {
 268                         *p++ = *q++;
 269 #ifdef DBG_TBUF
 270                         /* MR23 */ printMessage(stderr,
 271                         " '%s'", ((ANTLRCommonToken *)buffer[i-1])->getText());
 272 #endif
 273                 }
 274 #ifdef DBG_TBUF
 275                 /* MR23 */ printMessage(stderr, " ]\n");
 276 #endif
 277                 next = &buffer[k-1];
 278                 tp = &buffer[k-1];      // tp points to what will be filled in next
 279                 last = tp-1;
 280 #ifdef DBG_TBUF
 281                 /* MR23 */ printMessage(stderr,
 282                 "after: tp=%d, last=%d, next=%d\n",
 283                 tp-buffer, last-buffer, next-buffer);
 284 #endif
 285 /*
 286         }
 287         else {
 288                 extendBuffer();
 289         }
 290 */
 291 }
 292
 293 /* This function extends 'buffer' by chunk_size and returns with all
 294  * pointers at the same relative positions in the buffer (the buffer base
 295  * address could have changed in realloc()) except that 'next' comes
 296  * back set to where the next token should be stored.  All other pointers
 297  * are untouched.
 298  */
 299 void
 300 ANTLRTokenBuffer::
 301 extendBuffer()
 302 {
 303         int save_last = last-buffer, save_tp = tp-buffer, save_next = next-buffer;
 304 #ifdef DBG_TBUF
 305         /* MR23 */ printMessage(stderr, "extending physical buffer\n");
 306 #endif
 307         buffer_size += chunk_size;
 308         buffer = (_ANTLRTokenPtr *)
 309                 realloc((char *)(buffer-1),
 310                                 (buffer_size+1)*sizeof(_ANTLRTokenPtr ));
 311         if ( buffer == NULL ) {
 312                 panic("cannot alloc token buffer");
 313         }
 314         buffer++;                               // leave the first elem empty so tp-1 is valid ptr
 315
 316         tp = buffer + save_tp;  // put the pointers back to same relative position
 317         last = buffer + save_last;
 318         next = buffer + save_next;
 319         end_of_buffer = &buffer[buffer_size-1];
 320         threshold = &buffer[(int)(buffer_size*(1.0/2.0))];
 321
 322 /*
 323         // zero out new token ptrs so we'll know if something to delete in buffer
 324         ANTLRAbstractToken **p = end_of_buffer-chunk_size+1;
 325         for (; p<=end_of_buffer; p++) *p = NULL;
 326 */
 327 }
 328
 329 ANTLRParser * ANTLRTokenBuffer::                                // MR1
 330 setParser(ANTLRParser *p) {                                     // MR1
 331   ANTLRParser   *old=parser;                                    // MR1
 332   parser=p;                                                     // MR1
 333   input->setParser(p);                                          // MR1
 334   return old;                                                   // MR1
 335 }                                                               // MR1
 336                                                                 // MR1
 337 ANTLRParser * ANTLRTokenBuffer::                                // MR1
 338 getParser() {                                                   // MR1
 339   return parser;                                                // MR1
 340 }                                                               // MR1
 341
 342 void ANTLRTokenBuffer::panic(const char *msg) // MR23
 343 {
 344         if (parser)                             //MR23
 345                 parser->panic(msg);     //MR23
 346         else                                    //MR23
 347                 exit(PCCTS_EXIT_FAILURE);
 348 }
 349
 350 //MR23
 351 int ANTLRTokenBuffer::printMessage(FILE* pFile, const char* pFormat, ...)
 352 {
 353         va_list marker;
 354         va_start( marker, pFormat );
 355
 356         int iRet = 0;
 357         if (parser)
 358                 parser->printMessageV(pFile, pFormat, marker);
 359         else
 360                 iRet = vfprintf(pFile, pFormat, marker);
 361
 362         va_end( marker );
 363         return iRet;
 364 }
 365
 366 /* to avoid having to link in another file just for the smart token ptr
 367  * stuff, we include it here.  Ugh.
 368  *
 369  * MR23 This causes nothing but problems for IDEs.
 370  *      Change from .cpp to .h
 371  *
 372  */
 373
 374 #include ATOKPTR_IMPL_H