open_toontown_panda3d/direct/src/dcparser/dcLexer.lxx

/*
// Filename: dcLexer.lxx
// Created by:  drose (05Oct00)
// 
////////////////////////////////////////////////////////////////////
*/

%{
#include "dcLexerDefs.h"
#include "dcParserDefs.h"
#include "dcParser.h"
#include "dcFile.h"
#include "dcindent.h"


static int yyinput(void);        // declared by flex.
extern "C" int dcyywrap();

////////////////////////////////////////////////////////////////////
// Static variables
////////////////////////////////////////////////////////////////////

// We'll increment line_number and col_number as we parse the file, so
// that we can report the position of an error.
static int line_number = 0;
static int col_number = 0;

// current_line holds as much of the current line as will fit.  Its
// only purpose is for printing it out to report an error to the user.
static const int max_error_width = 1024;
static char current_line[max_error_width + 1];

static int error_count = 0;
static int warning_count = 0;

// This is the pointer to the current input stream.
static istream *inp = NULL;

// This is the name of the dc file we're parsing.  We keep it so we
// can print it out for error messages.
static string dc_filename;

// This is the initial token state returned by the lexer.  It allows
// the yacc grammar to start from initial points.
static int initial_token;


////////////////////////////////////////////////////////////////////
// Defining the interface to the lexer.
////////////////////////////////////////////////////////////////////

void
dc_init_lexer(istream &in, const string &filename) {
  inp = ∈
  dc_filename = filename;
  line_number = 0;
  col_number = 0;
  error_count = 0;
  warning_count = 0;
  initial_token = START_DC;
}

void
dc_start_parameter_value() {
  /* Set the initial state to begin parsing a parameter value, instead
     of at the beginning of the dc file. */
  initial_token = START_PARAMETER_VALUE;
}

void
dc_start_parameter_description() {
  /* Set the initial state to begin parsing a parameter description, instead
     of at the beginning of the dc file. */
  initial_token = START_PARAMETER_DESCRIPTION;
}

int
dc_error_count() {
  return error_count;
}

int
dc_warning_count() {
  return warning_count;
}


////////////////////////////////////////////////////////////////////
// Internal support functions.
////////////////////////////////////////////////////////////////////

int
dcyywrap(void) {
  return 1;
}

void
dcyyerror(const string &msg) {
  cerr << "\nError";
  if (!dc_filename.empty()) {
    cerr << " in " << dc_filename;
  }
  cerr 
    << " at line " << line_number << ", column " << col_number << ":\n"
    << current_line << "\n";
  indent(cerr, col_number-1) 
    << "^\n" << msg << "\n\n";
  
  error_count++;
}

void
dcyywarning(const string &msg) {
  cerr << "\nWarning";
  if (!dc_filename.empty()) {
    cerr << " in " << dc_filename;
  }
  cerr 
    << " at line " << line_number << ", column " << col_number << ":\n"
    << current_line << "\n";
  indent(cerr, col_number-1) 
    << "^\n" << msg << "\n\n";

  warning_count++;
}

// Now define a function to take input from an istream instead of a
// stdio FILE pointer.  This is flex-specific.
static void
input_chars(char *buffer, int &result, int max_size) {
  nassertv(inp != NULL);
  if (*inp) {
    inp->read(buffer, max_size);
    result = inp->gcount();
    if (result >= 0 && result < max_size) {
      // Truncate at the end of the read.
      buffer[result] = '\0';
    }

    if (line_number == 0) {
      // This is a special case.  If we are reading the very first bit
      // from the stream, copy it into the current_line array.  This
      // is because the \n.* rule below, which fills current_line
      // normally, doesn't catch the first line.
      strncpy(current_line, dcyytext, max_error_width);
      current_line[max_error_width] = '\0';
      line_number++;
      col_number = 0;

      // Truncate it at the newline.
      char *end = strchr(current_line, '\n');
      if (end != NULL) {
        *end = '\0';
      }
    }

  } else {
    // End of file or I/O error.
    result = 0;
  }
}
#undef YY_INPUT
#define YY_INPUT(buffer, result, max_size) input_chars(buffer, result, max_size)

// read_char reads and returns a single character, incrementing the
// supplied line and column numbers as appropriate.  A convenience
// function for the scanning functions below.
static int
read_char(int &line, int &col) {
  int c = yyinput();
  if (c == '\n') {
    line++;
    col = 0;
  } else {
    col++;
  }
  return c;
}

// scan_quoted_string reads a string delimited by quotation marks and
// returns it.
static string
scan_quoted_string(char quote_mark) {
  string result;

  // We don't touch the current line number and column number during
  // scanning, so that if we detect an error while scanning the string
  // (e.g. an unterminated string), we'll report the error as
  // occurring at the start of the string, not at the end--somewhat
  // more convenient for the user.

  // Instead of adjusting the global line_number and col_number
  // variables, we'll operate on our own local variables for the
  // interim.
  int line = line_number;
  int col = col_number;

  int c;
  c = read_char(line, col);
  while (c != quote_mark && c != EOF) {
    // A newline is not allowed within a string unless it is escaped.
    if (c == '\n') {
      c = EOF;
      break;

    } else if (c == '\\') {
      // Backslash escapes the following character.  We also respect
      // some C conventions.
      c = read_char(line, col);
      switch (c) {
      case 'a':
        result += '\a';
        c = read_char(line, col);
        break;

      case 'n':
        result += '\n';
        c = read_char(line, col);
        break;

      case 'r':
        result += '\r';
        c = read_char(line, col);
        break;

      case 't':
        result += '\t';
        c = read_char(line, col);
        break;

      case 'x':
        {
          int hex = 0;
          c = read_char(line, col);
          for (int i = 0; i < 2 && isxdigit(c); i++) {
            hex = hex * 16 + (isdigit(c) ? c - '0' : tolower(c) - 'a' + 10);
            c = read_char(line, col);
          }

          result += hex;
        }
        break;

      case '0':
        {
          int oct = 0;
          c = read_char(line, col);
          for (int i = 0; i < 3 && (c >= '0' && c < '7'); i++) {
            oct = oct * 8 + (c - '0');
            c = read_char(line, col);
          }

          result += oct;
        }
        break;

      case '1':
      case '2':
      case '3':
      case '4':
      case '5':
      case '6':
      case '7':
      case '8':
      case '9':
        {
          int dec = 0;
          c = read_char(line, col);
          for (int i = 0; i < 3 && isdigit(c); i++) {
            dec = dec * 10 + (c - '0');
            c = read_char(line, col);
          }

          result += dec;
        }
        break;

      case EOF:
        break;

      default:
        result += c;
        c = read_char(line, col);
      }

    } else {
      result += c;
      c = read_char(line, col);
    }
  }

  if (c == EOF) {
    dcyyerror("This quotation mark is unterminated.");
  }

  line_number = line;
  col_number = col;

  return result;
}

// scan_hex_string reads a string of hexadecimal digits delimited by
// angle brackets and returns the representative string.
static string
scan_hex_string() {
  string result;

  // We don't touch the current line number and column number during
  // scanning, so that if we detect an error while scanning the string
  // (e.g. an unterminated string), we'll report the error as
  // occurring at the start of the string, not at the end--somewhat
  // more convenient for the user.

  // Instead of adjusting the global line_number and col_number
  // variables, we'll operate on our own local variables for the
  // interim.
  int line = line_number;
  int col = col_number;

  bool odd = false;
  int last = 0;
  int c;
  c = read_char(line, col);
  while (c != '>' && c != EOF) {
    int value;
    if (c >= '0' && c <= '9') {
      value = c - '0';
    } else if (c >= 'a' && c <= 'f') {
      value = c - 'a' + 10;
    } else if (c >= 'A' && c <= 'F') {
      value = c - 'A' + 10;
    } else {
      line_number = line;
      col_number = col;
      dcyyerror("Invalid hex digit.");
      return string();
    }

    odd = !odd;
    if (odd) {
      last = value;
    } else {
      result += (char)((last << 4) | value);
    }
    c = read_char(line, col);
  }

  if (c == EOF) {
    dcyyerror("This hex string is unterminated.");
    return string();
  } else if (odd) {
    dcyyerror("Odd number of hex digits.");
    return string();
  }

  line_number = line;
  col_number = col;

  return result;
}

// eat_c_comment scans past all characters up until the first */
// encountered.
static void
eat_c_comment() {
  // As above, we'll operate on our own local copies of line_number
  // and col_number within this function.

  int line = line_number;
  int col = col_number;

  int c, last_c;
  
  last_c = '\0';
  c = read_char(line, col);
  while (c != EOF && !(last_c == '*' && c == '/')) {
    if (last_c == '/' && c == '*') {
      dcyywarning("This comment contains a nested /* symbol--possibly unclosed?");
    }
    last_c = c;
    c = read_char(line, col);
  }

  if (c == EOF) {
    dcyyerror("This comment marker is unclosed.");
  }

  line_number = line;
  col_number = col;
}



// accept() is called below as each piece is pulled off and
// accepted by the lexer; it increments the current column number.
inline void accept() {
  col_number += yyleng;
}

%}

UNSIGNED_INTEGERNUM  ([0-9]+)
SIGNED_INTEGERNUM    ([+-]([0-9]+))
UNSIGNED_HEXNUM      (0x[0-9a-fA-F]*)
REALNUM              ([+-]?(([0-9]+[.])|([0-9]*[.][0-9]+))([eE][+-]?[0-9]+)?)

%%

%{
  if (initial_token != 0) {
    int t = initial_token;
    initial_token = 0;
    return t;
  }
%}

\n.* {
  // New line.  Save a copy of the line so we can print it out for the
  // benefit of the user in case we get an error.

  strncpy(current_line, dcyytext+1, max_error_width);
  current_line[max_error_width] = '\0';
  line_number++;
  col_number=0;

  // Return the whole line to the lexer, except the newline character,
  // which we eat.
  yyless(1);
}

[ \t\r] { 
  // Eat whitespace.
  accept();
}

"//".* { 
  // Eat C++-style comments.
  accept();
}

"/*" {
  // Eat C-style comments.
  accept();
  eat_c_comment(); 
}


"dclass" {
  accept();
  return KW_DCLASS;
}

"struct" {
  accept();
  return KW_STRUCT;
}

"from" {
  accept();
  return KW_FROM;
}

"import" {
  accept();
  return KW_IMPORT;
}

"keyword" {
  accept();
  return KW_KEYWORD;
}

"typedef" {
  accept();
  return KW_TYPEDEF;
}

"switch" {
  accept();
  return KW_SWITCH;
}

"case" {
  accept();
  return KW_CASE;
}

"default" {
  accept();
  return KW_DEFAULT;
}

"break" {
  accept();
  return KW_BREAK;
}

"int8" {
  accept();
  return KW_INT8;
}

"int16" {
  accept();
  return KW_INT16;
}

"int32" {
  accept();
  return KW_INT32;
}

"int64" {
  accept();
  return KW_INT64;
}

"uint8" {
  accept();
  return KW_UINT8;
}

"uint16" {
  accept();
  return KW_UINT16;
}

"uint32" {
  accept();
  return KW_UINT32;
}

"uint64" {
  accept();
  return KW_UINT64;
}

"float64" {
  accept();
  return KW_FLOAT64;
}

"string" {
  accept();
  return KW_STRING;
}

"blob" {
  accept();
  return KW_BLOB;
}

"blob32" {
  accept();
  return KW_BLOB32;
}

"int8array" {
  accept();
  return KW_INT8ARRAY;
}

"int16array" {
  accept();
  return KW_INT16ARRAY;
}

"int32array" {
  accept();
  return KW_INT32ARRAY;
}

"uint8array" {
  accept();
  return KW_UINT8ARRAY;
}

"uint16array" {
  accept();
  return KW_UINT16ARRAY;
}

"uint32array" {
  accept();
  return KW_UINT32ARRAY;
}

"uint32uint8array" {
  accept();
  return KW_UINT32UINT8ARRAY;
}

"char" {
  accept();
  return KW_CHAR;
}

{UNSIGNED_INTEGERNUM} { 
  // An unsigned integer number.
  accept();

  // atoll isn't fully portable, so we'll decode the integer by hand.
  dcyylval.str = dcyytext;
  dcyylval.u.uint64 = 0;
  const char *p = dcyytext;
  while (*p != '\0') {
    PN_uint64 next_value = dcyylval.u.uint64 * 10;
    if (next_value < dcyylval.u.uint64) {
      dcyyerror("Number out of range.");
      dcyylval.u.uint64 = 1;
      return UNSIGNED_INTEGER;
    }

    dcyylval.u.uint64 = next_value + (*p - '0');
    ++p;
  }
  
  return UNSIGNED_INTEGER;
}

{SIGNED_INTEGERNUM} { 
  // A signed integer number.
  accept();

  // atoll isn't fully portable, so we'll decode the integer by hand.
  dcyylval.str = dcyytext;

  bool neg = false;
  const char *p = dcyytext;
  if (*p == '-') {
    neg = true;
    ++p;
  } else if (*p == '+') {
    ++p;
  }

  PN_uint64 value = 0;
  while (*p != '\0') {
    PN_uint64 next_value = value * 10;
    if (next_value < value) {
      dcyyerror("Number out of range.");
      dcyylval.u.int64 = 1;
      return SIGNED_INTEGER;
    }

    value = next_value + (*p - '0');
    ++p;
  }

  if (neg) {
    dcyylval.u.int64 = -(PN_int64)value;
    if (dcyylval.u.int64 > 0) {
      dcyyerror("Number out of range.");
      dcyylval.u.int64 = 1;
    }
  } else {
    dcyylval.u.int64 = (PN_int64)value;
    if (dcyylval.u.int64 < 0) {
      dcyyerror("Number out of range.");
      dcyylval.u.int64 = 1;
    }
  }    
  
  return SIGNED_INTEGER;
}

{UNSIGNED_HEXNUM} {
  // A hexadecimal integer number.
  accept(); 

  // As above, we'll decode the hex string by hand.
  dcyylval.str = dcyytext;
  dcyylval.u.uint64 = 0;
  const char *p = dcyytext + 2;
  while (*p != '\0') {
    PN_uint64 next_value = dcyylval.u.uint64 * 16;
    if (next_value < dcyylval.u.uint64) {
      dcyyerror("Number out of range.");
      dcyylval.u.uint64 = 1;
      return UNSIGNED_INTEGER;
    }

    if (isalpha(*p)) {
      dcyylval.u.uint64 = next_value + (tolower(*p) - 'a' + 10);
    } else {
      dcyylval.u.uint64 = next_value + (*p - '0');
    }
    ++p;
  }

  return UNSIGNED_INTEGER; 
}

{REALNUM} { 
  // A floating-point number.
  accept(); 
  dcyylval.u.real = atof(dcyytext); 
  dcyylval.str = dcyytext;
  return REAL; 
}

["] {
  // Quoted string.
  accept();
  dcyylval.str = scan_quoted_string('"');
  return STRING;
}

['] {
  // Single-quoted string.
  accept();
  dcyylval.str = scan_quoted_string('\'');
  return STRING;
}

[<] {
  // Long hex string.
  accept();
  dcyylval.str = scan_hex_string();
  return HEX_STRING;
}

[A-Za-z_][A-Za-z_0-9]* { 
  // Identifier or keyword.
  accept();
  dcyylval.str = dcyytext;

  if (dc_file != (DCFile *)NULL) {
    const DCKeyword *keyword = dc_file->get_keyword_by_name(dcyylval.str);
    if (keyword != (DCKeyword *)NULL) {
      dcyylval.u.keyword = keyword;
      return KEYWORD;
    }
  }
  return IDENTIFIER;
}


. {
  // Send any other printable character as itself.
  accept(); 
  return dcyytext[0];
}