-export function tokenize(input : string){
- var input_matchee_pair : Maybe<MatcheePair> = toSome(
- {matched:"",
- remained: input});
-
- // integer = ([+]|[-])?\d\d*
- let integer = (x : MatcheePair) =>
- { let wrapped_x = toSome(x);
- let plusMinus = orDo(match1Char('+'), match1Char('-')); // ([+]|[-])
- let d = matchRange('0','9'); // \d
- var result = thenDo(thenDo(thenDo(wrapped_x,
- zeroOrOnceDo(plusMinus)),d),
- zeroOrMoreDo(d));
-
- if (result._tag == "Some"){
- result.value.matched_type = TokenType.INT;
- }
- return result;
- }
- let space = (x : MatcheePair) =>{
- let wrapped_x = toSome(x);
- let s_aux = orDo(match1Char(' '), match1Char('\t')); // (" " | "\t")
- var result = thenDo(thenDo(wrapped_x, s_aux), zeroOrMoreDo(s_aux));
- if (result._tag == "Some"){
- result.value.matched_type = TokenType.SP;
- }
- return result;
- }
- let newline = (x : MatcheePair) =>{
- let wrapped_x = toSome(x);
- // nl = \r?\n
- let result = thenDo(thenDo(wrapped_x,
- zeroOrOnceDo(match1Char('\r'))), match1Char('\n'));
- if (result._tag == "Some"){
- result.value.matched_type = TokenType.NL;
- }
- return result;
- }
-
- let term = (token_list : Array<Token>, x : Some<MatcheePair>)=>{
- var ln = 1;
- var col = 0;
- var old_x = x;
- let term_list = [newline, space, integer];
- let term_aux = term_list.reduce((x,y)=> orDo(x,y));