/ray/src/lib/lex/iscannerbase.cc

Go to the documentation of this file.

/*
 * flex/iscannerbase.cc
 * 
 * Internal base class for flex-based scanner base class. 
 * 
 * Copyright (c) 2003--2004 by Wolfgang Wieser ] wwieser (a) gmx <*> de [ 
 * 
 * This file may be distributed and/or modified under the terms of the 
 * GNU General Public License version 2 as published by the Free Software 
 * Foundation. (See COPYING.GPL for details.)
 * 
 * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
 * WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
 * 
 */

#include <lib/sconfig.h>    /* must be first */

#include <lib/serror.h>
#include <lib/tl/linkedlist.h>
#include <lib/lex/location.h>
#include <lib/sourcepos/sparchive.h>
#include <lib/tl/tlstring.h>
#include <lib/message/message.h>

#include <unistd.h>  /* USE STREAMS INSTEAD!! */
#include <stdio.h>   /* USE STREAMS INSTEAD!! */
#include <errno.h>   /* USE STREAMS INSTEAD!! */

// For the implementation of the internal base class, we use no 
// _InternalFlexScannerBase, so that this does not get included. 
#undef _InternalFlexScannerBase
#include "scannerbase.h"


//---------------------------------<Config>-------------------------------------

_InternalFlexScannerBase::Config::Config()
{
    // Defaults for values which are not changed by _reset(): 
    loc_use_lpos=1;
    tab_width=8;
    max_file_depth=32;
    ahead_toks_wanted=1;   // minimum 1
    old_toks_wanted=16;
}

//-------------------------------<TokenEntry>-----------------------------------

bool _InternalFlexScannerBase::TokenEntry::MayBeCleared() const
{
    // This is always true for pure simple tokens (i.e. no allocated 
    // pointers inside). 
    return(1);
}

void _InternalFlexScannerBase::TokenEntry::clear(bool /*force*/)
{
    // Nothing special to do here; however clearing the state makes 
    // debugging easier...
    token=-1;
    lloc=SCLocationRange();
}


_InternalFlexScannerBase::TokenEntry *_InternalFlexScannerBase::
	AllocTokenEntry()
{
    return new TokenEntry();
}


//-------------------------------<LexerInput>-----------------------------------

ssize_t _InternalFlexScannerBase::LexerInput::read(char *buf,size_t len)
{
    // FIXME: To be replaced with stream implementation and no stderr,.. error. 
    
    if(fp)
    {
        errno=0;
        size_t result;
        if(feof(fp))
        {  result=0;  }
        else while( (result = fread(buf,1,len,fp))==0 && ferror(fp) )
        {
            if(errno!=EINTR)
            {
                fprintf(stderr,"reading %s failed: %s\n",
                    fp_path.str(),strerror(errno));
                return(-1);
                break;
            }
            errno=0;
            clearerr(fp);
        }
        
        //fprintf(stderr,"read()=%d\n",result);
        return(result);
    }
    
    return(0);
}


int _InternalFlexScannerBase::LexerInput::OpenFile(const TLString &path,
    SError &error)
{
    // FIXME: To be replaced with stream implementation. 
    
    if(fp)  return(1);
    
    if(path)
    {
        fp=fopen(path.str(),"r");
        if(!fp)
        {  /* FIXME: set error*/ error=SError("Open failed blah blah",1); return(-2);  }
        
        fp_path=path;
    }
    else
    {
        fp=stdin;
        fp_path="[stdin]";
    }
    
    return(0);
}


_InternalFlexScannerBase::LexerInput::LexerInput(LexerInput *_down) : 
    down(_down),
    fp(NULL),
    fp_path(),
    saved(-1,-1),
    must_pop_state(0),
    must_pop_pos_arch(0),
    read_eof(0),
    special_next_tok(-1),
    next_toks(),
    prev_toks()
{
    // Nothing more to do. 
}

_InternalFlexScannerBase::LexerInput::~LexerInput()
{
    if(fp)
    {
        if(fp!=stdin) fclose(fp);
        fp=NULL;
    }
    
    while(!next_toks.IsEmpty())
    {
        Error("OOPS: ~ALSInput: Token %d left in next_toks.",
            next_toks.first()->token);
        next_toks.first()->clear(/*force=*/1);
        delete next_toks.PopFirst();
    }
    
    // Tokens with MayBeCleared()==0 from this list must have been 
    // requeued in zombie_list before. 
    while(!prev_toks.IsEmpty())
    {  delete prev_toks.PopFirst();  }
}

//-------------------------------<SCLocation>-----------------------------------

SCLocation _InternalFlexScannerBase::_MakeCurrLoc() const
{
    return(SCLocation(pos_arch->GetPos(p0.line,
        cfg.loc_use_lpos ? p0.lpos : -1)));
}

SCLocation _InternalFlexScannerBase::_MakeCurrEndLoc() const
{
    return(SCLocation(pos_arch->GetPos(p1.line,
        cfg.loc_use_lpos ? p1.lpos : -1)));
}

SCLocationRange _InternalFlexScannerBase::_MakeCurrLocRange() const
{
    return(SCLocationRange(_MakeCurrLoc(),_MakeCurrEndLoc()));
}

//----------------------------<Lexer Interface>---------------------------------

void _InternalFlexScannerBase::_pi_char(char c)
{
    switch(c)
    {
        case '\n':  _pi_line();  break;
        case '\r':  break;   // should not happen due to filtering
        case '\t':  _pi_tab();  break;
        default:  ++p1.lpos;  break;
    }
}

void _InternalFlexScannerBase::_pi_str(const char *str,size_t len)
{
    for(const char *c=str,*cend=str+len; c<cend; c++)
    {  _pi_char(*c);  }
}

void _InternalFlexScannerBase::_pi_eof()
{
    Assert(top_inp);
    top_inp->read_eof=1;
    //fprintf(stderr,"_pi_eof()\n");
}


//---------------------------<_InternalFlexScannerBase>-------------------------


int _InternalFlexScannerBase::SetInput(const TLString &path,SError &error,
    bool included,
    int first_token,XYPos alt_pos,SourcePositionArchive *alt_pos_arch)
{
    Assert(magic==MyMagic);
    
    // Input validation: 
    if(included && alt_pos_arch)  return(-3);
    if(scanner && !included)  return(1);
    
    if(included && file_depth>=cfg.max_file_depth)  return(-4);
    
    // Set up LexerInput; "down" pointer set further down. 
    LexerInput *inp=new LexerInput(/*down=*/NULL);
    if(inp->OpenFile(path,error))
    {  DELETE(inp);  return(-2);  }
    
    // Set special next token if specified: 
    if(first_token>=0)
    {  inp->special_next_tok=first_token;  }
    
    // Do scanner setup (if needed): 
    if(!scanner)
    {  _lex_init(&scanner);  }  // will set extra pointer
    
    int rv=_SetInputBH(inp,included,alt_pos,alt_pos_arch);
    
    // Make sure we know which input is on top: 
    inp->down=top_inp;
    top_inp=inp;
    
    return(rv);
}


_InternalFlexScannerBase::TokenEntry *_InternalFlexScannerBase::LexNextToken()
{
    if(!scanner)  return(NULL);
    
    for(;;)
    {
        cont2:;
        
        /* WARNING: DO NOT TOUCH THAT CODE UNLESS YOU **REALLY** KNOW */
        /*          WHAT YOU ARE DOING.  SOME CALLS HAVE UNEXPECTED   */
        /*          SIDE EFFECTS (esp. when processing #includes).    */
        /*                                 -- Wolfgang Jul/03; Dez/04 */
        
        Assert(magic==MyMagic);
        Assert(top_inp);
        
        if(!read_inp)
        {  read_inp=top_inp;  }
        
        // Okay, I maintain a token buffer for some additional 
        // lookahead and look-back if needed. 
        LexerInput *from_inp=NULL;
        
        // In case read_inp!=top_inp. 
        // The lexer is in a different input than we are. 
        // This is due to some #include. 
        // Must first return all tokens from the current read_inp. 
        if(read_inp!=top_inp)
        {
            /* See if we have tokens: */
            if(read_inp->next_toks.IsEmpty())
            {
                // No toks left; go on: 
                read_inp=top_inp;
            }
            else
            {  from_inp=read_inp;  }
        } /* NO ELSE. */
        if(read_inp==top_inp)
        {
            // Make sure token buffer is full... 
            LexerInput *inp=top_inp;
            // I call count() here because for 2 or 3 tokens this 
            // costs "no" time. 
            int n_next=inp->next_toks.count();
            int n_prev=inp->prev_toks.count();
            // Want to have ahead_toks_wanted in the next_toks list. 
            // Get new ones unless we read EOF. 
            while(n_next<cfg.ahead_toks_wanted && !inp->read_eof)
            {
                // Get TokenEntry to use: 
                TokenEntry *te=NULL;
                if(n_prev>cfg.old_toks_wanted)
                {
                    te=inp->prev_toks.PopFirst();
                    --n_prev;
                    // Read comment about zombie_list...
                    if(te->MayBeCleared())
                    {  te->clear();  } // Get rid of old content. 
                    else
                    {
                        zombie_list.append(te);
                        ++zombie_list_nents;
                        te=NULL;
                    }
                }
                if(!te)
                {  te=AllocTokenEntry();  }
                
                // Do the lexical analysis & queue token: 
                bool state_change=0;
                _RawYYLex(te);
                if(inp!=top_inp)   // <-- May have changed...
                {  inp=top_inp;  state_change=1;  }
                inp->next_toks.append(te);
                ++n_next;
                
                // If scanner state changed, go back to beginning: 
                if(state_change)
                {  goto cont2;  }
            }
            while(n_prev>cfg.old_toks_wanted)
            {
                TokenEntry *tmp=inp->prev_toks.PopFirst();
                --n_prev;
                // Read comment about zombie_list...
                if(tmp->MayBeCleared())
                {  delete tmp;  }
                else
                {  zombie_list.append(tmp);  ++zombie_list_nents;  }
                // Should not happen unless we just #included something. 
                if(!inp->read_eof)
                {  Warning("HMMM: Del Old Tok.");  }
            }
            from_inp=inp;
        }
        
        _TidyUpZombieList();  // inline check
        
        // Get token to be returned: 
        TokenEntry *te=from_inp->next_toks.PopFirst();
        Assert(te);
        // If we have ahead_toks_wanted=1 (minimum value) it is important 
        // that we really do not read more tokens (we may need to switch 
        // context thus differently tokens may be recognized. 
        Assert(cfg.ahead_toks_wanted>1 || from_inp->next_toks.IsEmpty());
        // "Eat" token: 
        from_inp->prev_toks.append(te);
        
        // *te is the token to be returned. 
        //(FIXME) cloc=te->lloc;  /* another copy */
        
        if(te->token)   // Not EOF token. 
        {
            // You can plug in smart lookahead / context-sensitive 
            // de-ambiguation here. (FIXME)
            
            /*if(te->token<256)
            {  fprintf(stderr,"\tTOK=\'%c\'\t\t[%s]\n",te->token,
                from_inp->fp_path.str());  }
            else if(te->token==TS_IDENTIFIER)
            {  fprintf(stderr,"\tTOK=\"%s\"\t\t[%s]\n",te->lval.str_val,
                from_inp->fp_path.str());  }
            else
            {  fprintf(stderr,"\tTOK=%d\t\t[%s]\n",te->token,
                from_inp->fp_path.str());  }*/
            
            return(te);
        }
        
        // EOF token here. 
        // The EOF token is only returned if we cannot _wrap any more. 
        // First, copy the data, becuase _wrap() will delete 
        // the current LexerInput and thus the token buffer and *te. 
        // --> Already done above.  
        
        read_inp=NULL;
        if(_wrap())
        {
            // Real EOF reached. 
            break;
        }
    }
    
    return(NULL);   // EOF
}


void _InternalFlexScannerBase::_RawYYLex(TokenEntry *dest)
{
    Assert(magic==MyMagic);
    
    // "Shift" up position...
    p0=p1;  // C++ assignment
    
    // Do the actual lex'ing: 
    int tok_rv;
    if(top_inp->special_next_tok>=0)
    {
        tok_rv=top_inp->special_next_tok;
        top_inp->special_next_tok=-1;
    }
    else
    {
        lex_token=dest;
        tok_rv=_lex(scanner);
        lex_token=NULL;
    }
    
    // Store the token: 
    dest->token=tok_rv;
    if(!tok_rv)  Assert(top_inp->read_eof || n_errors);
    
    // Store the position: 
    dest->lloc=_MakeCurrLocRange();
    
    /*if(rv<256)
    {  fprintf(stderr,"   LEX=\'%c\'\n",rv);  }
    else
    {  fprintf(stderr,"   LEX=%d\n",tok_rv);  }*/
}


/* NOTE: Always use _TidyUpZombieList() and not the "__" version. */
void _InternalFlexScannerBase::__TidyUpZombieList(bool force)
{
    // In order to avoid too many tokens in the zombie list and OTOH 
    // not checking all tokens all the time, the zombie_list_threash is 
    // used. If there are at least zombie_list_threash new entries in 
    // the zombie list, the last 2*zombie_list_threash tokens (max.) 
    // will be checked for tid up. This means, that tokens are asked 
    // twice or a bit more often but not "all the time". 
    // 
    // This approach seems natural for use together with a shift/reduce 
    // parser where some tokens may spend a long time on the stack until 
    // finally getting used. 
    
    int _cnt=0,_nforce=0,_ntry=0;
    int ncheck = force ? -1 : 2*zombie_list_threash;
    for(TokenEntry *_i=zombie_list.last(); _i && ncheck; ncheck--)
    {
        TokenEntry *i=_i;
        _i=_i->prev;
        
        bool we_may=i->MayBeCleared();
        if(force && !we_may)
        {  ++_nforce;  we_may=1;  }
        
        if(we_may)
        {
            i->clear(/*force=*/force);
            
            delete zombie_list.dequeue(i);
            --zombie_list_nents;
            ++_cnt;
        }
        
        ++_ntry;
    }
    
    // Getting this message is perfectly normal. 
    if(_cnt<_ntry || (force && (_cnt || _nforce)))
    {  Warning(
        "Hmmm: deleted %d (%d forced, %d tried) zombie tokens, %d left "
        "(force=%d).",
        _cnt,_nforce,_ntry,zombie_list_nents,int(force));  }
    if(force)
    {  Assert(zombie_list_nents==0);  }
    
    Assert(zombie_list_nents==zombie_list.count());
    last_zombie_list_nents=zombie_list_nents;
}


void _InternalFlexScannerBase::_reset(bool no_virtuals)
{
    // NOTE: May not call virtual functions which go "below" 
    // PREFIX_FlexScannerBase (as called from destructor). 
    Assert(magic==MyMagic);
    
    // Must be NULL if outside yylex(). 
    Assert(!lex_token);
    
    if(scanner)
    {
        // Tidy up the stack: 
        read_inp=NULL;
        while(!_wrap());
        
        Assert(!file_depth);
        Assert(!top_inp);
        
        _lex_destroy(scanner);
        scanner=NULL;
    }
    
    p0.line=1;  p1.line=1;
    p0.lpos=0;  p1.lpos=0;
    
    if(pos_arch)
    {
        if(pos_arch_allocated)
        {  delete pos_arch;  }
        pos_arch=NULL;
        pos_arch_allocated=0;
    }
    
    _TidyUpZombieList(/*force=*/1);
    
    n_errors=0;
    
    top_inp=NULL;
    read_inp=NULL;
    file_depth=0;
    
    // Call virtual function for derived class unless forbidden. 
    if(!no_virtuals)
    {  reset();  }
}


void _InternalFlexScannerBase::reset()
{
    // Virtual function which only has to be overridden if the derived 
    // parser needs that. Nothing to do for default implementation. 
}


_InternalFlexScannerBase::_InternalFlexScannerBase() : 
    zombie_list(),
    p0(1,0),
    p1(1,0),
    cfg()
{
    magic=MyMagic;
    scanner=NULL;
    lex_token=NULL;
    
    top_inp=NULL;
    read_inp=NULL;
    
    zombie_list_nents=0;
    last_zombie_list_nents=0;
    
    pos_arch=NULL;
    pos_arch_allocated=0;
    
    // All the other values are initialized in _reset(). 
    // This will be called from within PREFIX_FlexScannerBase. 
    // _reset();
}

_InternalFlexScannerBase::~_InternalFlexScannerBase()
{
    // The following is called from within PREFIX_FlexScannerBase 
    // due to virtual function calls. 
    //_InternalFlexScannerBase::_reset();
    
    Assert(!scanner);
    
    Assert(!top_inp);
    Assert(!read_inp);
    
    Assert(magic==MyMagic);
    magic=0;
}

//------------------------------------------------------------------------------

int _InternalFlexScannerBase::_lex(void * /*scanner*/)
{
    // Must be overridden if used. 
    CritAssert(0);  // override
}

int _InternalFlexScannerBase::_lex_init(void ** /*scanner_p*/)
{
    // Must be overridden if used. 
    CritAssert(0);  // override
}

void _InternalFlexScannerBase::_lex_destroy(void * /*scanner*/)
{
    // Must be overridden if used. 
    CritAssert(0);  // override
}

int _InternalFlexScannerBase::_wrap()
{
    // Must be overridden if used. 
    CritAssert(0);  // override
    return(0);
}

int _InternalFlexScannerBase::_SetInputBH(LexerInput * /*inp*/,
    bool /*included*/,
    XYPos /*alt_pos*/,SourcePositionArchive * /*alt_pos_arch*/)
{
    // Must be overridden if used. 
    CritAssert(0);  // override
    return(-5);
}

---