/ray/src/lib/lex/test-scanner.ll

Go to the documentation of this file.

%{
/*
 * lib/lex/test-scanner.ll
 * 
 * Test and demo scanner. 
 * 
 * Copyright (c) 2002--2004 by Wolfgang Wieser ] wwieser (a) gmx <*> de [ 
 * 
 * This file may be distributed and/or modified under the terms of the 
 * GNU General Public License version 2 as published by the Free Software 
 * Foundation. (See COPYING.GPL for details.)
 * 
 * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
 * WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
 * 
 */


#include <lib/message/message.h>

#include <lib/lex/test-scanner.h>

#include <lib/lex/test-grammar.h>


#define YY_EXTRA_TYPE  class TestScanner*

// Special "trick": include scannerbase.cc for the first time: 
#include "scannerbase.cc"

// Get pointer where to store token info (string/integer/float val...)
#define TOK()    ((TestScanner::TestToken*)(yyextra->_tok_ptr()))

%}

/* Generate scanner file by using: 
 * flex -s test-scanner.ll    [works with flex >= 2.5.10] */

%pointer
%option 8bit
%option reentrant
/*%option bison-bridge*/
/* option bison-locations does NOT calculate locations; just passes 
 * the structure to store the pos. Then, I don't need it. */
/*%option bison-locations*/
/*%option yylineno*/
%option nodefault
%option noyywrap
%option stack
%option prefix="TEST_"
%option outfile="test-scanner.cc"
/*%option header-file="test-scanner.hh"*/

    /*IN_CMT    -> C-style comment   */
    /*IN_CPPCMT -> C++-style comment */
%x IN_CMT
%x IN_CPPCMT

identifier  [[:alpha:]_][[:alnum:]_]*
ninteger    [[:digit:]]+
xinteger    (0?[[:digit:]]+)|(0x[[:xdigit:]]+)

%%

"//"  {
        PI_TOK();
        yy_push_state(IN_CPPCMT,yyscanner);
    }
<IN_CPPCMT>.*\n  {
        PI_LINE();
        PI_START();
        yy_pop_state(yyscanner);
    }
<IN_CPPCMT><<EOF>>  {
        PI_EOF();
        PI_TOK();
        PI_START();
        yy_pop_state(yyscanner);
        return(0);
    }
    /* This rule will normally not match (only if terminated by EOF). */
<IN_CPPCMT>.  {
        PI_CHAR(yytext[0]);
    }

"/*"  {
        PI_TOK();
        yy_push_state(IN_CMT,yyscanner);
        yyextra->nested_comment_warned=0;
        yyextra->newlines_in_ccomment=0;
    }
<IN_CMT>"*/"  {
        PI_TOK();
        PI_START();
        yy_pop_state(yyscanner);
    }
<IN_CMT>"/*"  {
        PI_START();  /* BEFORE PI_TOK here. */
        PI_TOK();
        if(yyextra->allow_nested_comments)
        {
            if(yyextra->allow_nested_comments>=2 && 
               !yyextra->nested_comment_warned)
            {
                Warning(yyextra->_MakeCurrLoc(),"nested C-style comment\n");
                ++yyextra->nested_comment_warned;
            }
            yy_push_state(IN_CMT,yyscanner);
        }
        PI_START();
    }
<IN_CMT>[^\n/\*]*\n    {  PI_LINE();  ++yyextra->newlines_in_ccomment;  }
<IN_CMT>[^\n\r\t/\*]*  {  PI_TOK();  }
    /* The following rule cannot be matched. */
    /* <IN_CMT>\n     {  PI_LINE();  ++yyextra->newlines_in_ccomment;  } */
<IN_CMT>\t             {  PI_TAB();  }
<IN_CMT><<EOF>>  {
        PI_EOF();
        PI_TOK();
        /* FIXME: we do not yet report where the unterminated comment starts!!! */
        Error(yyextra->_MakeCurrEndLoc(),"unterminated C-style comment\n");
        ++yyextra->n_errors;
        while(YY_START==IN_CMT)
        {  yy_pop_state(yyscanner);  }
        return(0);
    }
<IN_CMT>.              {  PI_TOK();  /* single '*' would fail otherwise */  }

    /* Match whitespace: */
[ \t]+  {
        PI_STR(yytext,yyleng);
        /* As this is not a token, re-set token start pos: */
        PI_START();
    }
\n  {  PI_LINE();  PI_START();  }


"if"      {  PI_TOK();  return(TS_IF);  }
"else"    {  PI_TOK();  return(TS_ELSE);  }

{xinteger}  {
        PI_TOK();
        // Parse integer value: 
        int ival=strtol(yytext,NULL,0);
        // Set result in token: 
        TOK()->lval.int_val=ival;
        // Return "token". 
        return(TS_INTEGER);
    }
{identifier}  {
        PI_TOK();
        // "Parse" identifier (no-op) and store in token value: 
        TOK()->lval.string_val=ALLOC<char>(yyleng+1);
        strncpy(TOK()->lval.string_val,yytext,yyleng);
        TOK()->lval.string_val[yyleng]='\0';
        return(TS_IDENTIFIER);
    }

<<EOF>>  {
        /* The normal action would be to YYWarp() here but I cannot */
        /* because of the token buffer. */
        /*if(yyextra->_YYWrap())*/
        /*{  return(0);  }*/
        PI_EOF();
        return(0);
    }

.   {  PI_TOK();  return(yytext[0]);  }

%%

// Special "trick"; 2nd time; see comment in scannerbase.h. 
#define PREFIX_FlexScannerBase TEST_FlexScannerBase
#define PREFIX_lex TEST_lex
#define PREFIX_lex_init TEST_lex_init
#define PREFIX_lex_destroy TEST_lex_destroy
#define PREFIX_pop_buffer_state TEST_pop_buffer_state
#define PREFIX_push_buffer_state TEST_push_buffer_state
#define PREFIX_create_buffer TEST__create_buffer
#define PREFIX_switch_to_buffer TEST__switch_to_buffer
inline void PREFIX_set_extra(PREFIX_FlexScannerBase *arg,yyscan_t scanner)
    {  yyset_extra((TestScanner*)arg,scanner);  }
#define FLEX_BOTTOM_PART
#include "scannerbase.cc"


void TestScanner::reset()
{
    nested_comment_warned=0;
    newlines_in_ccomment=0;
}


TestScanner::TestScanner() : 
    TEST_FlexScannerBase()
{
    nested_comment_warned=0;
    newlines_in_ccomment=0;
    
    allow_nested_comments=1;
}

TestScanner::~TestScanner()
{
    
}

//------------------------------------------------------------------------------

bool TestScanner::TestToken::MayBeCleared() const
{
    if(token!=TS_IDENTIFIER)  return(1);
    // Check magic indication that the token was used: 
    if(!lval.string_val || *lval.string_val=='\0')  return(1);
    return(0);
}

void TestScanner::TestToken::clear(bool force)
{
    // We need to deallocate the string_val if necessary; 
    // this is only the case for the TS_IDENTIFIER token. 
    if(!force)
    {  Assert(MayBeCleared());  }
    if(token==TS_IDENTIFIER)
    {  lval.string_val=FREE(lval.string_val);  }
    
    // Important: call higher level: 
    TokenEntry::clear();
}

---