rebuild parser

[clo] / src / tokenize.ts

diff --git a/src/tokenize.ts b/src/tokenize.ts
index 861b638e32420fc41b677eb943cd24d47eff601a..144dbedc0736f5eae1da0063ad9869e321e59d7c 100644 (file)
--- a/src/tokenize.ts
+++ b/src/tokenize.ts
@@ -1,3 +1,4 @@
+import * as util from 'util';
 
 var fs = require('fs');
 
@@ -63,7 +64,7 @@ export interface MatcheePair {
  * SEMI_C// semi-colon
  */
 export enum TokenType {
-    NL, // newlinw
+    NL, // newline
     SP, // half-width space and tab
     ID, // identifier
     STR, // string
@@ -98,7 +99,9 @@ export enum TokenType {
     NE, // <>
     APOS, // '
     R_ARROW, // ->
-
+    TRUE, // true
+    FALSE, // false
+    IF, // if
 }
 
 /**
@@ -197,6 +200,25 @@ export function matchRange(l: string, u: string): (m: MatcheePair) => Maybe<Matc
     }
 };
 
+
+/**
+ * check if a matcheePair `m` matches a stringv `s`. 
+ * @param s the checker string.
+ * @returns `None` or matched pair wrapped in `Some`
+ */
+export function matchWord(s: string, ): (m: MatcheePair) => Maybe<MatcheePair> {
+    return (m)=>{
+        if (s.length==0){
+            return { _tag: "None" };
+        }
+        var someM : Maybe<MatcheePair> = toSome(m);
+        for (var idx : number=0; idx<s.length; idx++){
+            someM = thenDo(someM, match1Char(s[idx]))
+        }
+        return someM;
+    }
+}
+
 /**
  * convert the one-char string to codepoint.
  * @param s : the string to code point.
@@ -356,7 +378,7 @@ export function tokenize(input: string): Array<Token> {
     // space = [ \t]+
     let space = bTerm((x: Maybe<MatcheePair>) =>
         thenDo(thenDo(x, s_aux), zeroOrMoreDo(s_aux)),
-        TokenType.INT);
+        TokenType.SP);
 
     // newline = \r?\n
     let newline = bTerm((x: Maybe<MatcheePair>) =>
@@ -444,6 +466,7 @@ export function tokenize(input: string): Array<Token> {
         thenDo(thenDo(x, match1Char("-")), match1Char(">")),
         TokenType.R_ARROW);
 
+
     /**
      * unary operator : generating the pattern of basic unary operator
      * @param char : uniry char for the operator
@@ -488,7 +511,7 @@ export function tokenize(input: string): Array<Token> {
             lParen, rParen, lBracket, rBracket, lBrace, rBrace,
             comma, dot, colon, semicolon, at, hash,
             set, greaterthan, lessthan, apos,
-            float, newline, space, integer, str, id];
+            float, newline, space,  id,  integer, str];
         let term_aux = term_list.reduce((x, y) => orDo(x, y));
 
         var new_x: Maybe<MatcheePair> = thenDo(old_x, term_aux);