- * Symbols currently returned include:
- * Symbol Constant Returned Symbol Constant Returned
- * ------ ----------------- ------ -----------------
- * "package" PACKAGE "import" IMPORT
- * "code" CODE "action" ACTION
- * "parser" PARSER "terminal" TERMINAL
- * "non" NON "init" INIT
- * "scan" SCAN "with" WITH
- * "start" START "precedence" PRECEDENCE
- * "left" LEFT "right" RIGHT
- * "nonassoc" NONASSOC "%prec PRECENT_PREC
- * [ LBRACK ] RBRACK
- * ; SEMI
- * , COMMA * STAR
- * . DOT : COLON
- * ::= COLON_COLON_EQUALS | BAR
- * identifier ID {:...:} CODE_STRING
- * "nonterminal" NONTERMINAL
- *
- * All symbol constants are defined in sym.java which is generated by
- * JavaCup from parser.cup.- * - * In addition to the scanner proper (called first via init() then with - * next_token() to get each Symbol) this class provides simple error and - * warning routines and keeps a count of errors and warnings that is - * publicly accessible.
- * - * This class is "static" (i.e., it has only static members and methods). - * - * @version last updated: 7/3/96 - * @author Frank Flannery - */ -public class lexer { - - /*-----------------------------------------------------------*/ - /*--- Constructor(s) ----------------------------------------*/ - /*-----------------------------------------------------------*/ - - /** The only constructor is private, so no instances can be created. */ - private lexer() { } - - /*-----------------------------------------------------------*/ - /*--- Static (Class) Variables ------------------------------*/ - /*-----------------------------------------------------------*/ - - /** First character of lookahead. */ - protected static int next_char; - - /** Second character of lookahead. */ - protected static int next_char2; - - /** Second character of lookahead. */ - protected static int next_char3; - - /** Second character of lookahead. */ - protected static int next_char4; - - /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ - - /** EOF constant. */ - protected static final int EOF_CHAR = -1; - - /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ - - /** Table of keywords. Keywords are initially treated as identifiers. - * Just before they are returned we look them up in this table to see if - * they match one of the keywords. The string of the name is the key here, - * which indexes Integer objects holding the symbol number. - */ - protected static Hashtable keywords = new Hashtable(23); - - /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ - - /** Table of single character symbols. For ease of implementation, we - * store all unambiguous single character Symbols in this table of Integer - * objects keyed by Integer objects with the numerical value of the - * appropriate char (currently Character objects have a bug which precludes - * their use in tables). - */ - protected static Hashtable char_symbols = new Hashtable(11); - - /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ - - /** Current line number for use in error messages. */ - protected static int current_line = 1; - - /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ - - /** Character position in current line. */ - protected static int current_position = 1; - - /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ - - /** Character position in current line. */ - protected static int absolute_position = 1; - - /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ - - /** Count of total errors detected so far. */ - public static int error_count = 0; - - /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ - - /** Count of warnings issued so far */ - public static int warning_count = 0; - - /*-----------------------------------------------------------*/ - /*--- Static Methods ----------------------------------------*/ - /*-----------------------------------------------------------*/ - - /** Initialize the scanner. This sets up the keywords and char_symbols - * tables and reads the first two characters of lookahead. - */ - public static void init() throws java.io.IOException - { - /* set up the keyword table */ - keywords.put("package", new Integer(sym.PACKAGE)); - keywords.put("import", new Integer(sym.IMPORT)); - keywords.put("code", new Integer(sym.CODE)); - keywords.put("action", new Integer(sym.ACTION)); - keywords.put("parser", new Integer(sym.PARSER)); - keywords.put("terminal", new Integer(sym.TERMINAL)); - keywords.put("non", new Integer(sym.NON)); - keywords.put("nonterminal",new Integer(sym.NONTERMINAL));// [CSA] - keywords.put("init", new Integer(sym.INIT)); - keywords.put("scan", new Integer(sym.SCAN)); - keywords.put("with", new Integer(sym.WITH)); - keywords.put("start", new Integer(sym.START)); - keywords.put("precedence", new Integer(sym.PRECEDENCE)); - keywords.put("left", new Integer(sym.LEFT)); - keywords.put("right", new Integer(sym.RIGHT)); - keywords.put("nonassoc", new Integer(sym.NONASSOC)); - - /* set up the table of single character symbols */ - char_symbols.put(new Integer(';'), new Integer(sym.SEMI)); - char_symbols.put(new Integer(','), new Integer(sym.COMMA)); - char_symbols.put(new Integer('*'), new Integer(sym.STAR)); - char_symbols.put(new Integer('.'), new Integer(sym.DOT)); - char_symbols.put(new Integer('|'), new Integer(sym.BAR)); - char_symbols.put(new Integer('['), new Integer(sym.LBRACK)); - char_symbols.put(new Integer(']'), new Integer(sym.RBRACK)); - - /* read two characters of lookahead */ - next_char = System.in.read(); - if (next_char == EOF_CHAR) { - next_char2 = EOF_CHAR; - next_char3 = EOF_CHAR; - next_char4 = EOF_CHAR; - } else { - next_char2 = System.in.read(); - if (next_char2 == EOF_CHAR) { - next_char3 = EOF_CHAR; - next_char4 = EOF_CHAR; - } else { - next_char3 = System.in.read(); - if (next_char3 == EOF_CHAR) { - next_char4 = EOF_CHAR; - } else { - next_char4 = System.in.read(); - } - } - } - } - - /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ - - /** Advance the scanner one character in the input stream. This moves - * next_char2 to next_char and then reads a new next_char2. - */ - protected static void advance() throws java.io.IOException - { - int old_char; - - old_char = next_char; - next_char = next_char2; - if (next_char == EOF_CHAR) { - next_char2 = EOF_CHAR; - next_char3 = EOF_CHAR; - next_char4 = EOF_CHAR; - } else { - next_char2 = next_char3; - if (next_char2 == EOF_CHAR) { - next_char3 = EOF_CHAR; - next_char4 = EOF_CHAR; - } else { - next_char3 = next_char4; - if (next_char3 == EOF_CHAR) { - next_char4 = EOF_CHAR; - } else { - next_char4 = System.in.read(); - } - } - } - - /* count this */ - absolute_position++; - current_position++; - if (old_char == '\n' || (old_char == '\r' && next_char!='\n')) - { - current_line++; - current_position = 1; - } - } - - /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ - - /** Emit an error message. The message will be marked with both the - * current line number and the position in the line. Error messages - * are printed on standard error (System.err). - * @param message the message to print. - */ - public static void emit_error(String message) - { - System.err.println("Error at " + current_line + "(" + current_position + - "): " + message); - error_count++; - } - - /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ - - /** Emit a warning message. The message will be marked with both the - * current line number and the position in the line. Messages are - * printed on standard error (System.err). - * @param message the message to print. - */ - public static void emit_warn(String message) - { - System.err.println("Warning at " + current_line + "(" + current_position + - "): " + message); - warning_count++; - } - - /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ - - /** Determine if a character is ok to start an id. - * @param ch the character in question. - */ - protected static boolean id_start_char(int ch) - { - /* allow for % in identifiers. a hack to allow my - %prec in. Should eventually make lex spec for this - frankf */ - return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || - (ch == '_'); - - // later need to deal with non-8-bit chars here - } - - /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ - - /** Determine if a character is ok for the middle of an id. - * @param ch the character in question. - */ - protected static boolean id_char(int ch) - { - return id_start_char(ch) || (ch >= '0' && ch <= '9'); - } - - /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ - - /** Try to look up a single character symbol, returns -1 for not found. - * @param ch the character in question. - */ - protected static int find_single_char(int ch) - { - Integer result; - - result = (Integer)char_symbols.get(new Integer((char)ch)); - if (result == null) - return -1; - else - return result.intValue(); - } - - /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ - - /** Handle swallowing up a comment. Both old style C and new style C++ - * comments are handled. - */ - protected static void swallow_comment() throws java.io.IOException - { - /* next_char == '/' at this point */ - - /* is it a traditional comment */ - if (next_char2 == '*') - { - /* swallow the opener */ - advance(); advance(); - - /* swallow the comment until end of comment or EOF */ - for (;;) - { - /* if its EOF we have an error */ - if (next_char == EOF_CHAR) - { - emit_error("Specification file ends inside a comment"); - return; - } - - /* if we can see the closer we are done */ - if (next_char == '*' && next_char2 == '/') - { - advance(); - advance(); - return; - } - - /* otherwise swallow char and move on */ - advance(); - } - } - - /* is its a new style comment */ - if (next_char2 == '/') - { - /* swallow the opener */ - advance(); advance(); - - /* swallow to '\n', '\r', '\f', or EOF */ - while (next_char != '\n' && next_char != '\r' && - next_char != '\f' && next_char!=EOF_CHAR) - advance(); - - return; - - } - - /* shouldn't get here, but... if we get here we have an error */ - emit_error("Malformed comment in specification -- ignored"); - advance(); - } - - /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ - - /** Swallow up a code string. Code strings begin with "{:" and include - all characters up to the first occurrence of ":}" (there is no way to - include ":}" inside a code string). The routine returns a String - object suitable for return by the scanner. - */ - protected static Symbol do_code_string() throws java.io.IOException - { - StringBuffer result = new StringBuffer(); - - /* at this point we have lookahead of "{:" -- swallow that */ - advance(); advance(); - - /* save chars until we see ":}" */ - while (!(next_char == ':' && next_char2 == '}')) - { - /* if we have run off the end issue a message and break out of loop */ - if (next_char == EOF_CHAR) - { - emit_error("Specification file ends inside a code string"); - break; - } - - /* otherwise record the char and move on */ - result.append(new Character((char)next_char)); - advance(); - } - - /* advance past the closer and build a return Symbol */ - advance(); advance(); - return new Symbol(sym.CODE_STRING, result.toString()); - } - - /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ - - /** Process an identifier. Identifiers begin with a letter, underscore, - * or dollar sign, which is followed by zero or more letters, numbers, - * underscores or dollar signs. This routine returns a String suitable - * for return by the scanner. - */ - protected static Symbol do_id() throws java.io.IOException - { - StringBuffer result = new StringBuffer(); - String result_str; - Integer keyword_num; - char buffer[] = new char[1]; - - /* next_char holds first character of id */ - buffer[0] = (char)next_char; - result.append(buffer,0,1); - advance(); - - /* collect up characters while they fit in id */ - while(id_char(next_char)) - { - buffer[0] = (char)next_char; - result.append(buffer,0,1); - advance(); - } - - /* extract a string and try to look it up as a keyword */ - result_str = result.toString(); - keyword_num = (Integer)keywords.get(result_str); - - /* if we found something, return that keyword */ - if (keyword_num != null) - return new Symbol(keyword_num.intValue()); - - /* otherwise build and return an id Symbol with an attached string */ - return new Symbol(sym.ID, result_str); - } - - /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ - - /** Return one Symbol. This is the main external interface to the scanner. - * It consumes sufficient characters to determine the next input Symbol - * and returns it. To help with debugging, this routine actually calls - * real_next_token() which does the work. If you need to debug the - * parser, this can be changed to call debug_next_token() which prints - * a debugging message before returning the Symbol. - */ - public static Symbol next_token() throws java.io.IOException - { - return real_next_token(); - } - - /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ - - /** Debugging version of next_token(). This routine calls the real scanning - * routine, prints a message on System.out indicating what the Symbol is, - * then returns it. - */ - public static Symbol debug_next_token() throws java.io.IOException - { - Symbol result = real_next_token(); - System.out.println("# next_Symbol() => " + result.sym); - return result; - } - - /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ - - /** The actual routine to return one Symbol. This is normally called from - * next_token(), but for debugging purposes can be called indirectly from - * debug_next_token(). - */ - protected static Symbol real_next_token() throws java.io.IOException - { - int sym_num; - - for (;;) - { - /* look for white space */ - if (next_char == ' ' || next_char == '\t' || next_char == '\n' || - next_char == '\f' || next_char == '\r') - { - /* advance past it and try the next character */ - advance(); - continue; - } - - /* look for a single character symbol */ - sym_num = find_single_char(next_char); - if (sym_num != -1) - { - /* found one -- advance past it and return a Symbol for it */ - advance(); - return new Symbol(sym_num); - } - - /* look for : or ::= */ - if (next_char == ':') - { - /* if we don't have a second ':' return COLON */ - if (next_char2 != ':') - { - advance(); - return new Symbol(sym.COLON); - } - - /* move forward and look for the '=' */ - advance(); - if (next_char2 == '=') - { - advance(); advance(); - return new Symbol(sym.COLON_COLON_EQUALS); - } - else - { - /* return just the colon (already consumed) */ - return new Symbol(sym.COLON); - } - } - - /* find a "%prec" string and return it. otherwise, a '%' was found, - which has no right being in the specification otherwise */ - if (next_char == '%') { - advance(); - if ((next_char == 'p') && (next_char2 == 'r') && (next_char3 == 'e') && - (next_char4 == 'c')) { - advance(); - advance(); - advance(); - advance(); - return new Symbol(sym.PERCENT_PREC); - } else { - emit_error("Found extraneous percent sign"); - } - } - - /* look for a comment */ - if (next_char == '/' && (next_char2 == '*' || next_char2 == '/')) - { - /* swallow then continue the scan */ - swallow_comment(); - continue; - } - - /* look for start of code string */ - if (next_char == '{' && next_char2 == ':') - return do_code_string(); - - /* look for an id or keyword */ - if (id_start_char(next_char)) return do_id(); - - /* look for EOF */ - if (next_char == EOF_CHAR) return new Symbol(sym.EOF); - - /* if we get here, we have an unrecognized character */ - emit_warn("Unrecognized character '" + - new Character((char)next_char) + "'(" + next_char + - ") -- ignored"); - - /* advance past it */ - advance(); - } - } - - /*-----------------------------------------------------------*/ -} -