-
-export function tokenize(input : string){
- var input_matchee_pair : Maybe<MatcheePair> = toSome(
- {matched:"",
- remained: input});
-
- // integer = ([+]|[-])?\d\d*
- let integer = (x : MatcheePair) =>
- { let wrapped_x = toSome(x);
- let plusMinus = orDo(match1Char('+'), match1Char('-')); // ([+]|[-])
- let d = matchRange('0','9'); // \d
- var result = thenDo(thenDo(thenDo(wrapped_x,
- zeroOrOnceDo(plusMinus)),d),
- zeroOrMoreDo(d));
-
- if (result._tag == "Some"){
- result.value.matched_type = TokenType.INT;
- }
- return result;
- }
- let space = (x : MatcheePair) =>{
- let wrapped_x = toSome(x);
- let s_aux = orDo(match1Char(' '), match1Char('\t')); // (" " | "\t")
- var result = thenDo(thenDo(wrapped_x, s_aux), zeroOrMoreDo(s_aux));
- if (result._tag == "Some"){
- result.value.matched_type = TokenType.SP;
- }
- return result;
- }
- let newline = (x : MatcheePair) =>{
- let wrapped_x = toSome(x);
- // nl = \r?\n
- let result = thenDo(thenDo(wrapped_x,
- zeroOrOnceDo(match1Char('\r'))), match1Char('\n'));
- if (result._tag == "Some"){
- result.value.matched_type = TokenType.NL;
- }
- return result;
- }
-
- let term = (token_list : Array<Token>, x : Some<MatcheePair>)=>{
- var ln = 1;
- var col = 0;
- var old_x = x;
- let term_list = [newline, space, integer];
- let term_aux = term_list.reduce((x,y)=> orDo(x,y));
-
- var new_x : Maybe<MatcheePair> = thenDo(old_x, term_aux);
- while (new_x._tag != "None"){
- if (new_x.value.matched_type != TokenType.NL){
- col += new_x.value.matched.length;
- token_list.push({text : new_x.value.matched,
- type: new_x.value.matched_type,
- ln : ln,
- col : col});
-
- }
- else{
- col = 0;
- ln += 1;
-
- token_list.push({text : new_x.value.matched,
- type: new_x.value.matched_type,
- ln : ln,
- col : col});