X-Git-Url: https://git.kianting.info/?a=blobdiff_plain;f=src%2Findex.js;h=efa8b6854087eff3ca3b68153f1a575b614d0dba;hb=5fbb467bdf501dee03c78817242581d962547c4f;hp=712af45015c8dbd4747f9df53f1506724ca8effd;hpb=13c9e7026b3a5ba274ef046db3311b76aaaac244;p=clo diff --git a/src/index.js b/src/index.js index 712af45..efa8b68 100644 --- a/src/index.js +++ b/src/index.js @@ -1,7 +1,41 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); -exports.thenDo = exports.charToCodepoint = exports.matchRange = exports.match1Char = void 0; +exports.tokenize = exports.zeroOrOnceDo = exports.notDo = exports.zeroOrMoreDo = exports.orDo = exports.thenDo = exports.charToCodepoint = exports.matchRange = exports.matchAny = exports.match1Char = exports.TokenType = void 0; var fs = require('fs'); +/** + * wrap a x in a `Some(T)` + * @param x : variable to be wrapped. + * @returns wrapped `x`. + */ +function toSome(x) { + return { _tag: "Some", value: x }; +} +/** + * The types of Token + * NL, // newline + * + * SP, // half-width space and tab + * + * ID, // identifier + * + * STR, // string + * + * OP, // operator or something like it + * + * FLO, // float num + * + * INT, // Integer + */ +var TokenType; +(function (TokenType) { + TokenType[TokenType["NL"] = 0] = "NL"; + TokenType[TokenType["SP"] = 1] = "SP"; + TokenType[TokenType["ID"] = 2] = "ID"; + TokenType[TokenType["STR"] = 3] = "STR"; + TokenType[TokenType["OP"] = 4] = "OP"; + TokenType[TokenType["FLO"] = 5] = "FLO"; + TokenType[TokenType["INT"] = 6] = "INT"; +})(TokenType || (exports.TokenType = TokenType = {})); /** * @description * it returns a function which test if the first char of the `remained` part of @@ -12,6 +46,9 @@ var fs = require('fs'); */ function match1Char(c) { return (m) => { + if (m.remained.length == 0) { + return { _tag: "None" }; + } const charToBeMatched = m.remained[0]; if (charToBeMatched === c) { return { _tag: "Some", value: { @@ -26,6 +63,24 @@ function match1Char(c) { } exports.match1Char = match1Char; ; +/** + * + * @param m : the `MatcheePair` to be consumed. + * @returns if the length of `m.remained` >= 1; consumes the matchee by 1 char and wraps it in `Some`, + * otherwise, returns `None`. + */ +function matchAny(m) { + if (m.remained.length >= 1) { + return { _tag: "Some", value: { + matched: m.matched + m.remained[0], + remained: m.remained.substring(1) + } }; + } + else { + return { _tag: "None" }; + } +} +exports.matchAny = matchAny; /** * @description * it returns a function which test if the first char of the `remained` part of @@ -42,6 +97,9 @@ function matchRange(l, u) { throw new Error("Error: the codepoint of `" + l + "` is not smaller than `" + u + "`)"); } return (m) => { + if (m.remained.length < 1) { + return { _tag: "None" }; + } const charToBeMatched = m.remained[0]; const codePointToBeMatched = charToCodepoint(charToBeMatched); if (codePointToBeMatched >= lCodepoint && codePointToBeMatched <= uCodepoint) { @@ -75,6 +133,10 @@ exports.charToCodepoint = charToCodepoint; /** * @description thendo(input, f, ...) like * a ==> f + * @param input: the wrapped input. + * @param f: the function to be applied. + * + * @returns:the applied wrapped result `MatcheePair`. */ function thenDo(input, f) { if (input._tag == "None") { @@ -86,3 +148,159 @@ function thenDo(input, f) { } } exports.thenDo = thenDo; +/** + * @description "or", like the regex `( f1 | f2 )` . + * It returns a function `f` of which the argument is`x`. + * if `f1(x)` is None, then `f` returns `f2(x)`. Otherwise, + * `F` returns `f1(x)`. + * @param f1 : 1st function to be compared + * @param f2 : 2nd function to be compared + * @returns:the combined function + */ +function orDo(f1, f2) { + return (x) => { + let f1x = (f1(x)); + { + if (f1x._tag == "None") { + return f2(x); + } + else { + return f1x; + } + } + }; +} +exports.orDo = orDo; +/** +* @description repeating matching function `f` +* zero or more times, like the asterisk `*` in regex `f*` . +* @param f : the function to be repeated 0+ times. +* @returns:the combined function +*/ +function zeroOrMoreDo(f) { + return (x) => { + var wrapped_old_x = { _tag: "Some", value: x }; + var wrapped_new_x = wrapped_old_x; + while (wrapped_new_x._tag != "None") { + wrapped_old_x = wrapped_new_x; + wrapped_new_x = thenDo(wrapped_old_x, f); + } + ; + return wrapped_old_x; + }; +} +exports.zeroOrMoreDo = zeroOrMoreDo; +/** +* @description Not. like the `^` inside regex of [^f]. +* returns a function `F(x)` such that if `f(x)` is `None`, +* returns the x consuming a char; if `f(x)` is not None, F(x) +* returns `None`. +* @param f: the function forbidden to be matched. +* @returns: combined function `F`. +*/ +function notDo(f) { + return (x) => { + let wrapped_x = { + _tag: "Some", + value: x + }; + let f_x = thenDo(wrapped_x, f); + if (f_x._tag != "None") { + return { _tag: "None" }; + } + else { + return thenDo(wrapped_x, matchAny); + } + }; +} +exports.notDo = notDo; +/** + * if `x` is matched by `f` once, returns `f(x)`. Otherwise, + * returns x + * similar to `?` in regex `f?`. + * @param f : the function to be matched + * @returns return wrapped f(x) + */ +function zeroOrOnceDo(f) { + return (x) => { + var wrapped_old_x = { _tag: "Some", value: x }; + var wrapped_new_x = thenDo(wrapped_old_x, f); + if (wrapped_new_x._tag != "None") { + return wrapped_new_x; + } + else { + return wrapped_old_x; + } + }; +} +exports.zeroOrOnceDo = zeroOrOnceDo; +function tokenize(input) { + var input_matchee_pair = toSome({ matched: "", + remained: input }); + // integer = ([+]|[-])?\d\d* + let integer = (x) => { + let wrapped_x = toSome(x); + let plusMinus = orDo(match1Char('+'), match1Char('-')); // ([+]|[-]) + let d = matchRange('0', '9'); // \d + var result = thenDo(thenDo(thenDo(wrapped_x, zeroOrOnceDo(plusMinus)), d), zeroOrMoreDo(d)); + if (result._tag == "Some") { + result.value.matched_type = TokenType.INT; + } + return result; + }; + let space = (x) => { + let wrapped_x = toSome(x); + let s_aux = orDo(match1Char(' '), match1Char('\t')); // (" " | "\t") + var result = thenDo(thenDo(wrapped_x, s_aux), zeroOrMoreDo(s_aux)); + if (result._tag == "Some") { + result.value.matched_type = TokenType.SP; + } + return result; + }; + let newline = (x) => { + let wrapped_x = toSome(x); + // nl = \r?\n + let result = thenDo(thenDo(wrapped_x, zeroOrOnceDo(match1Char('\r'))), match1Char('\n')); + if (result._tag == "Some") { + result.value.matched_type = TokenType.NL; + } + return result; + }; + let term = (token_list, x) => { + var ln = 1; + var col = 0; + var old_x = x; + let term_list = [newline, space, integer]; + let term_aux = term_list.reduce((x, y) => orDo(x, y)); + var new_x = thenDo(old_x, term_aux); + while (new_x._tag != "None") { + if (new_x.value.matched_type != TokenType.NL) { + col += new_x.value.matched.length; + token_list.push({ text: new_x.value.matched, + type: new_x.value.matched_type, + ln: ln, + col: col }); + } + else { + col = 0; + ln += 1; + token_list.push({ text: new_x.value.matched, + type: new_x.value.matched_type, + ln: ln, + col: col }); + } + old_x = toSome({ matched: "", + remained: new_x.value.remained }); + new_x = thenDo(old_x, term_aux); + } + if (old_x.value.remained.length) { + console.log(token_list); + throw new Error("the code can't be tokenized is near Ln. " + ln + ", Col." + col + + ", starting with " + old_x.value.remained.substring(0, 10)); + } + return token_list; + }; + console.log(term([], input_matchee_pair)); + // TODO: id, string, space, basic operator, 3 marks: @, {, }. +} +exports.tokenize = tokenize;