title: printing tokens using lex and yacc
body: <p>I have a lex file , a yacc file and main.cpp file. </p>
  
  <p>My main.cpp looks like</p>
  
  <pre><code>int main(int argc, char **argv)
  {
      if (argc == 1)
      {   int token;
          curr_filename = "&lt;stdin&gt;";
          yyin = stdin;
          yyparse();
      }
      else
      {
          for (int i = 1; i &lt; argc; ++i)
          {
              curr_filename = argv[i];
              yyin = std::fopen(argv[i], "r");
  
              if (yyin)
              {    
  
                  yyparse();  
  
                  std::fclose(yyin);
              }
              else
              {
                  utility::print_error(argv[i], "cannot be opened");
              }
          }
      }
  
      if (yynerrs &gt; 0)
      {
          std::cerr &lt;&lt; "Compilation halted due to lexical or syntax errors.\n";
          exit(1);
      }
  </code></pre>
  
  <p>This is helping to do a parse .But now i want to print token generated from the lex file also. So i do a little amendment to it by calling yylex() as follows</p>
  
  <pre><code>    int main(int argc, char **argv)
      {
          if (argc == 1)
          {   int token;
              curr_filename = "&lt;stdin&gt;";
  
         yyin = stdin;
  // calling yylex to get token 
       while(token= yylex())
       {
          switch(token){
          case 258 : 
          std::cout &lt;&lt; "class" ;
          default : 
          std::cout &lt;&lt; "token " ;
  
                   }
  
  
              yyparse();
          }
  //rest of the code same
  </code></pre>
  
  <p>but nothing is getting printed to output. </p>
  
  <p>Any help how to get tokens printed on standard output or file </p>
  
  <p>flex file </p>
  
  <pre><code>%option noyywrap
  %option yylineno
  
  %{
  
  #include "flexbison.hpp"
  #include "tokentable.hpp"
  #include "symboltable.hpp"
  #include "y.tab.h"
  #include &lt;stdio.h&gt;
  
  #define YY_USER_ACTION yylloc.first_line = yylloc.last_line = yylineno; 
  
  static const int MAX_STR_CONST = 1025;
  
  char string_buf[MAX_STR_CONST];  // buffer to store string contstants encountered in source file
  char *string_buf_ptr;
  
  
  
  int num_comment = 0;      // count to keep track how many opening comment tokens have been encountered
  std::size_t curr_lineno = 0;      // keep track of current line number of source file
  bool str_too_long = false;   // used to handle string constant size error check
  
  
  %}
  
  %x COMMENT
  %x LINECOMMENT
  %x STRING
  
  DARROW =&gt;
  
  %%
  
  "(*" {
      BEGIN(COMMENT);
      num_comment++;
  }
  
  "*)" {
      if (num_comment &lt;= 0) {
          yylval.error_msg = "Unmatched *)";
          return ERROR;
      }
  }
  
  &lt;COMMENT&gt;"*)" {
      num_comment--;
      if (num_comment &lt; 0) {
          yylval.error_msg = "Unmatched *)";
          return ERROR;
      }
  
      if (num_comment == 0) {
          BEGIN(INITIAL);
      }
  }
  
  &lt;COMMENT&gt;"(*" {
      num_comment++;
  }
  
  &lt;COMMENT&gt;[^\n] {
      // eat everything within comments
  }
  
  &lt;COMMENT&gt;\n {
      ++curr_lineno;
  }
  
  "--"[^\n]* {
      BEGIN(LINECOMMENT);
  }
  
  &lt;LINECOMMENT&gt;\n {
      ++curr_lineno;
      BEGIN(INITIAL);
  }
  
  &lt;COMMENT&gt;&lt;&lt;EOF&gt;&gt; {
      BEGIN(INITIAL);
      yylval.error_msg = "EOF in comment";
      return ERROR;
  }
  
  "=&gt;" {
      return DARROW; 
  }
  
  (?i:class) {
      return CLASS;
  }
  
  (?i:else) {
      return ELSE;
  }
  
  (?i:in) {
      return IN;
  }
  
  (?i:then) {
      return THEN;
  }
  
  (?i:fi) {
      return FI;
  }
  
  (?i:if) {
      return IF;
  }
  
  (?i:inherits) {
      return INHERITS;
  }
  
  (?i:let) {
      return LET;
  }
  
  (?i:loop) {
      return LOOP;
  }
  
  (?i:pool) {
      return POOL;
  }
  
  (?i:while) {
      return WHILE;
  }
  
  (?i:case) {
      return CASE;
  }
  
  (?i:esac) {
      return ESAC;
  }
  
  (?i:of) {
      return OF;
  }
  
  (?i:new) {
      return NEW;
  }
  
  (?i:isvoid) {
      return ISVOID;
  }
  
  (?i:not) {
      return NOT;
  }
  
  t(?i:rue) {
      yylval.boolean = true;
      return BOOL_CONST;
  }
  
  f(?i:alse) {
      yylval.boolean = false;
      return BOOL_CONST;
  }
  
  [0-9]+ {
      yylval.symbol = inttable().add(yytext);
      return INT_CONST;
  }
  
  "&lt;=" {
      return LE;
  }
  
  "&lt;-" {
      return ASSIGN;
  }
  
  
  [A-Z][a-zA-Z0-9_]* {
      yylval.symbol = idtable().add(yytext);
      return TYPEID;
  }
  
  
  [a-z][a-zA-Z0-9_]* {
      yylval.symbol = idtable().add(yytext);
      return OBJECTID;
  }
  
  ";"|","|"{"|"}"|":"|"("|")"|"+"|"-"|"*"|"/"|"="|"~"|"&lt;"|"."|"@" { 
      return *yytext;
  }
  
  \n {
      ++curr_lineno;
  }
  
  [ \f\r\t\v] {
      // eat whitespace
  }
  
   /*
    *  String constants (C syntax)
    *  Escape sequence \c is accepted for all characters c. Except for 
    *  \n \t \b \f, the result is c.
    *
    */
  
  \" {
      BEGIN(STRING);
      string_buf_ptr = string_buf;
      memset(string_buf, 0, MAX_STR_CONST);
  }
  
  &lt;STRING&gt;\" {
      BEGIN(INITIAL);
      yylval.symbol = stringtable().add(string_buf);
      return STR_CONST;
  }
  
  &lt;STRING&gt;\0[^\n]*\" {
      BEGIN(INITIAL);
      if (str_too_long) {
          str_too_long = false;
      }
      else {
          yylval.error_msg = "String contains null character";
          return ERROR;
      }
  }
  
  &lt;STRING&gt;\0[^"]*\n {
      if (str_too_long) {
          yyinput(); /* eat quote */
          BEGIN(INITIAL);
          str_too_long = false;
      }
      else {
          if (yytext[yyleng - 1] != '\\') {
              BEGIN(INITIAL);
              yylval.error_msg = "String contains null character";
              return ERROR;
          }
      }
  }
  
  &lt;STRING&gt;&lt;&lt;EOF&gt;&gt; {
      BEGIN(INITIAL);
      yylval.error_msg = "EOF in string constant";
      return ERROR;
  }
  
  &lt;STRING&gt;\\ {
      if (strlen(string_buf) &gt;= MAX_STR_CONST - 1) {
          str_too_long = true;
          unput('\0');
          yylval.error_msg = "String constant too long";
          return ERROR;
      }
  
      char ahead = yyinput();
      switch (ahead) {
          case 'b':
              *string_buf_ptr++ = '\b';
              break;
          case 't':
              *string_buf_ptr++ = '\t';
              break;
          case 'n':
              *string_buf_ptr++ = '\n';
              break;
          case 'f':
              *string_buf_ptr++ = '\f';
              break;
          case '\n':
              ++curr_lineno;
              *string_buf_ptr++ = '\n';
              break;
          case '\0':
              unput(ahead);
              break;
          default:
              *string_buf_ptr++ = ahead;
      }
  }
  
  &lt;STRING&gt;\n {
      ++curr_lineno;
      BEGIN(INITIAL);
      yylval.error_msg = "Unterminated string constant";
      return ERROR;
  }
  
  &lt;STRING&gt;. {
      if (strlen(string_buf) &gt;= MAX_STR_CONST - 1) {
          str_too_long = true;
          unput('\0');
          yylval.error_msg = "String constant too long";
          return ERROR;
      }
  
      *string_buf_ptr++ = *yytext;
  }
  
  . /* error for invalid tokens */ {
      yylval.error_msg = std::string(yytext) + " is not a valid character in the current context.";
      return ERROR;
  }
  
  %%
  </code></pre>
  
  <p>bison file </p>
  
  <pre><code>%{
  
  #include "flexbison.hpp"
  #include "symboltable.hpp"
  #include "tokentable.hpp"
  #include "ast.hpp"
  
  #include &lt;iostream&gt;
  
  // convinience function for setting location of each ast node
  #define SETLOC(lval,node) (lval)-&gt;setloc((node).first_line, curr_filename)
  
  // both defined in main.cpp
  extern ProgramPtr ast_root;
  extern std::string curr_filename;
  
  // both defined in lexer
  extern int yylex();
  extern int yylineno;
  
  void yyerror(char *);        
  %}
  
  %token CLASS 258 ELSE 259 FI 260 IF 261 IN 262 
  %token INHERITS 263 LET 264 LOOP 265 POOL 266 THEN 267 WHILE 268
  %token CASE 269 ESAC 270 OF 271 DARROW 272 NEW 273 ISVOID 274
  %token &lt;symbol&gt;  STR_CONST 275 INT_CONST 276 
  %token &lt;boolean&gt; BOOL_CONST 277
  %token &lt;symbol&gt;  TYPEID 278 OBJECTID 279 
  %token ASSIGN 280 NOT 281 LE 282 ERROR 283
  
  %type &lt;program&gt; program
  %type &lt;clazz&gt; class
  %type &lt;classes&gt; class_list
  %type &lt;attribute&gt; attribute
  %type &lt;attributes&gt; attribute_list
  %type &lt;method&gt; method
  %type &lt;methods&gt; method_list
  %type &lt;expression&gt; expression
  %type &lt;expression&gt; let_expr 
  %type &lt;expressions&gt; expression_list
  %type &lt;expressions&gt; method_expr_list
  %type &lt;formal&gt; formal
  %type &lt;formals&gt; formal_list
  %type &lt;branch&gt; case
  %type &lt;cases&gt; case_list
  
  %nonassoc '='
  %left LET
  %right ASSIGN
  %left NOT
  %left '+' '-'
  %left '*' '/' 
  %left ISVOID
  %left '~'
  %left '@'
  %left '.'
  %nonassoc LE '&lt;'
  
  %%
  program : class_list    { @$ = @1; ast_root = std::make_shared&lt;Program&gt;($1); }
  ;
  
  class_list : class { $$ = Classes(); $$.push_back($1); }
              | class_list class { $$.push_back($2); }
  ;
  
  class : CLASS TYPEID '{' attribute_list method_list '}' ';' { $$ = std::make_shared&lt;Class&gt;($2, idtable().add("Object"), $4, $5); SETLOC($$, @1); }
          | CLASS TYPEID INHERITS TYPEID '{' attribute_list method_list '}' ';' { $$ = std::make_shared&lt;Class&gt;($2, $4, $6, $7); SETLOC($$, @1); }
          | error ';' { yyerrok; } 
  ;
  
  attribute_list : attribute ';' { $$ = Attributes(); $$.push_back($1); }
                 | attribute_list attribute ';' { $$.push_back($2); }
                 | error ';' { yyerrok; }
  ;
  
  attribute : OBJECTID ':' TYPEID { $$ = std::make_shared&lt;Attribute&gt;($1, $3, std::make_shared&lt;NoExpr&gt;()); SETLOC($$, @1); }
            | OBJECTID ':' TYPEID ASSIGN expression { $$ = std::make_shared&lt;Attribute&gt;($1, $3, $5); SETLOC($$, @5); }
  ;
  
  method_list : method ';' { $$ = Methods(); $$.push_back($1); }
              | method_list method ';' { $$.push_back($2); }
              | error ';' { yyerrok; }
  ;
  
  method : OBJECTID '(' formal_list ')' ':' TYPEID '{' expression '}' { $$ = std::make_shared&lt;Method&gt;($1, $6, $3, $8); SETLOC($$, @1); }
         | OBJECTID '(' ')' ':' TYPEID '{' expression '}' { $$ = std::make_shared&lt;Method&gt;($1, $5, Formals(), $7); SETLOC($$, @1); }
  ;
  
  formal_list : formal { $$ = Formals(); $$.push_back($1); }
              | formal_list ',' formal { $$.push_back($3); } 
  ;
  
  formal : OBJECTID ':' TYPEID { $$ = std::make_shared&lt;Formal&gt;($1, $3); SETLOC($$, @1); }
  ;
  
  case_list : case { $$ = Cases(); $$.push_back($1); }
              | case_list case { $$.push_back($2); }
  ;
  
  case : OBJECTID ':' TYPEID DARROW expression ';' { $$ = std::make_shared&lt;CaseBranch&gt;($1, $3, $5); SETLOC($$, @5); }
  ;
  
  method_expr_list : expression { $$ = Expressions(); $$.push_back($1); }
                      | method_expr_list ',' expression { $$.push_back($3); }
  ;
  
  expression_list : expression ';' { $$ = Expressions(); $$.push_back($1); }
                  | expression_list expression ';' { $$.push_back($2); }
                  | error ';' { yyerrok; }
  ;
  
  let_expr : OBJECTID ':' TYPEID IN expression %prec LET { $$ = std::make_shared&lt;Let&gt;($1, $3, std::make_shared&lt;NoExpr&gt;(), $5); SETLOC($$, @5); }
              | OBJECTID ':' TYPEID ASSIGN expression IN expression %prec LET { $$ = std::make_shared&lt;Let&gt;($1, $3, $5, $7); SETLOC($$, @5); }
              | OBJECTID ':' TYPEID ',' let_expr { $$ = std::make_shared&lt;Let&gt;($1, $3, std::make_shared&lt;NoExpr&gt;(), $5); SETLOC($$, @5); }
              | OBJECTID ':' TYPEID ASSIGN expression ',' let_expr { $$ = std::make_shared&lt;Let&gt;($1, $3, $5, $7); SETLOC($$, @4); }
              | error ',' let_expr { yyerrok; }
  ;
  
  
  expression : OBJECTID ASSIGN expression { $$ = std::make_shared&lt;Assign&gt;($1, $3); SETLOC($$, @3); }
              | expression '.' OBJECTID '(' method_expr_list ')' { $$ = std::make_shared&lt;DynamicDispatch&gt;($1, $3, $5); SETLOC($$, @1); }
              | expression '.' OBJECTID '(' ')' { $$ = std::make_shared&lt;DynamicDispatch&gt;($1, $3, Expressions()); SETLOC($$, @1); }
              | expression '@' TYPEID '.' OBJECTID '(' method_expr_list ')' { $$ = std::make_shared&lt;StaticDispatch&gt;($1, $3, $5, $7); SETLOC($$, @1); }
              | expression '@' TYPEID '.' OBJECTID '(' ')' { $$ = std::make_shared&lt;StaticDispatch&gt;($1, $3, $5, Expressions()); SETLOC($$, @1);}
              | OBJECTID '(' method_expr_list ')' { $$ = std::make_shared&lt;DynamicDispatch&gt;(std::make_shared&lt;Object&gt;(idtable().add("self")), $1, $3); 
                                                    SETLOC($$, @1); } 
              | OBJECTID '(' ')' { $$ = std::make_shared&lt;DynamicDispatch&gt;(std::make_shared&lt;Object&gt;(idtable().add("self")), $1, Expressions()); 
                                   SETLOC($$, @1); } 
              | IF expression THEN expression ELSE expression FI { $$ = std::make_shared&lt;If&gt;($2, $4, $6); SETLOC($$, @2); }
              | WHILE expression LOOP expression POOL { $$ = std::make_shared&lt;While&gt;($2, $4); SETLOC($$, @2); }
              | '{' expression_list '}' { $$ = std::make_shared&lt;Block&gt;($2); SETLOC($$, @2); }
              | LET let_expr { $$ = $2; SETLOC($$, @2); }
              | CASE expression OF case_list ESAC { $$ = std::make_shared&lt;Case&gt;($2, $4); SETLOC($$, @2); }
              | NEW TYPEID { $$ = std::make_shared&lt;New&gt;($2); SETLOC($$, @2); }
              | ISVOID expression { $$ = std::make_shared&lt;IsVoid&gt;($2); SETLOC($$, @2); }
              | expression '+' expression { $$ = std::make_shared&lt;Plus&gt;($1, $3); SETLOC($$, @1); }
              | expression '-' expression { $$ = std::make_shared&lt;Sub&gt;($1, $3); SETLOC($$, @1); }
              | expression '*' expression { $$ = std::make_shared&lt;Mul&gt;($1, $3); SETLOC($$, @1); }
              | expression '/' expression { $$ = std::make_shared&lt;Div&gt;($1, $3); SETLOC($$, @1); }
              | '~' expression { $$ = std::make_shared&lt;Complement&gt;($2); SETLOC($$, @2); }
              | expression '&lt;' expression { $$ = std::make_shared&lt;LessThan&gt;($1, $3); SETLOC($$, @1); }
              | expression LE expression { $$ = std::make_shared&lt;LessThanEqualTo&gt;($1, $3); SETLOC($$, @1); }
              | expression '=' expression { $$ = std::make_shared&lt;EqualTo&gt;($1, $3); SETLOC($$, @1); }
              | NOT expression { $$ = std::make_shared&lt;Not&gt;($2); SETLOC($$, @2); }
              | '(' expression ')' { $$ = $2; SETLOC($$, @2); } 
              | OBJECTID { $$ = std::make_shared&lt;Object&gt;($1); SETLOC($$, @1); }
              | INT_CONST { $$ = std::make_shared&lt;IntConst&gt;($1); SETLOC($$, @1); }
              | STR_CONST { $$ = std::make_shared&lt;StringConst&gt;($1); SETLOC($$, @1); }
              | BOOL_CONST { $$ = std::make_shared&lt;BoolConst&gt;($1); SETLOC($$, @1); } 
  ;
  
  %%
  
  // utility function for converting bison tokens to its string representation
  // for better error reporting
  std::string convert_token(int token)
  {
      std::string rep;
  
      switch (token)
      {
          case CLASS: rep = "class"; break;
          case ELSE: rep = "else"; break;
          case FI: rep = "fi"; break;
          case IF: rep = "if"; break;
          case IN: rep = "in"; break;
          case INHERITS: rep = "inherits"; break;
          case LET: rep = "let"; break;
          case LOOP: rep = "loop"; break;
          case POOL: rep = "pool"; break;
          case THEN: rep = "then"; break;
          case WHILE: rep = "while"; break;
          case CASE: rep = "case"; break;
          case ESAC: rep = "esac"; break;
          case OF: rep = "of"; break;
          case DARROW: rep = "=&gt;"; break;
          case NEW: rep = "new"; break;
          case ISVOID: rep = "isvoid"; break;
          case ASSIGN: rep = "&lt;-"; break;
          case NOT: rep = "not"; break;
          case LE: rep = "&lt;="; break;
          case STR_CONST: rep = "STR_CONST = " + yylval.symbol.get_val(); break;
          case INT_CONST: rep = "INT_CONST = " + yylval.symbol.get_val(); break;
          case BOOL_CONST: rep = "BOOL_CONST = " + yylval.boolean; break;
          case TYPEID: rep = "TYPEID = " + yylval.symbol.get_val(); break;
          case OBJECTID: rep = "OBJECTID = " + yylval.symbol.get_val(); break;
          default: rep = (char) token;
      }     
  
      return rep;
  }
  
  void yyerror(char *)
  {
      if (yylval.error_msg.length() &lt;= 0)
          std::cerr &lt;&lt; curr_filename &lt;&lt; ":" &lt;&lt; yylineno &lt;&lt; ": " &lt;&lt; "error: " &lt;&lt;  "syntax error near or at character or token '" &lt;&lt; convert_token(yychar) &lt;&lt; "'\n";
      else
          std::cerr &lt;&lt; curr_filename &lt;&lt; ":" &lt;&lt; yylineno &lt;&lt; ": " &lt;&lt; "error: " &lt;&lt; yylval.error_msg &lt;&lt; "\n";
  }
  </code></pre>