tagger benchmark

[IRC.git] / Robust / src / Benchmarks / mlp / tagger / original-java / src / tagger / Token.java

diff --git a/Robust/src/Benchmarks/mlp/tagger/original-java/src/tagger/Token.java b/Robust/src/Benchmarks/mlp/tagger/original-java/src/tagger/Token.java
new file mode 100755 (executable)
index 0000000..af03320
--- /dev/null
+++ b/Robust/src/Benchmarks/mlp/tagger/original-java/src/tagger/Token.java
@@ -0,0 +1,72 @@
+/**\r
+ * Token class\r
+ * Represents tokens generated by lexer\r
+ * <p>\r
+ *\r
+ * @author  Daniel Jackson\r
+ * @version 0, 07/06/01\r
+ */\r
+\r
+package tagger;\r
+import java.io.*;\r
+import java.util.*;\r
+\r
+public class Token {\r
+       // may be null\r
+       public String arg;\r
+       public int line;\r
+       public int type;\r
+\r
+       public static final int COMMENT = 0;\r
+       public static final int WHITESPACE = 1;\r
+       public static final int ALPHABETIC = 2;\r
+       public static final int NUMERIC = 3;\r
+\r
+       public static final int PARABREAK = 4;\r
+       public static final int LINEBREAK = 5;\r
+\r
+       public static final int APOSTROPHE = 10;\r
+       public static final int PRIME = 11;\r
+       public static final int OPENSINGLEQUOTE = 12;\r
+       public static final int CLOSESINGLEQUOTE = 13;\r
+       public static final int OPENDOUBLEQUOTE = 14;\r
+       public static final int CLOSEDOUBLEQUOTE = 15;\r
+       public static final int HYPHENS = 16;\r
+       public static final int DOTS = 17;\r
+\r
+       public static final int PARASTYLECOMMAND = 20;\r
+       public static final int FORMATCOMMAND = 21;\r
+       public static final int POPFORMATCOMMAND = 22;\r
+       public static final int REFCOMMAND = 23;\r
+       public static final int TAGCOMMAND = 24;\r
+       public static final int CITECOMMAND = 25;\r
+       public static final int CHARCOMMAND = 26;\r
+       public static final int LOADCHARMAPCOMMAND = 27;\r
+       public static final int LOADSTYLESHEETCOMMAND = 28;\r
+       public static final int PREAMBLECOMMAND = 29;\r
+       public static final int SEPARATORCOMMAND = 30;\r
+\r
+       // treated as a command for italicizing or as a char style\r
+       public static final int UNDERSCORE = 31;\r
+       public static final int DOLLAR = 32;\r
+       public static final int OTHER = 33;\r
+       public static final int ENDOFSTREAM = 34;\r
+\r
+       public static final int MAXTOKEN = 34;\r
+\r
+       public Token (int type, String arg, int line) {\r
+               this.type = type;\r
+               this.arg = arg;\r
+               this.line = line;\r
+               }\r
+\r
+       public Token (int type, int line) {\r
+               this.type = type;\r
+               this.line = line;\r
+               }\r
+\r
+       // temporary implementation\r
+       public String toString () {\r
+               return arg;\r
+               }\r
+}
\ No newline at end of file