X-Git-Url: https://git.kianting.info/?a=blobdiff_plain;f=src%2Findex.ts;h=405a1f0d465ffcf04867c6a2bb3c09c083c3b9b6;hb=ae71f2351bd6ffb5e054687dd1e3eb99f72545fe;hp=0d94c74c6fb2434f783f1451c0b86f3824973a80;hpb=1a7598fd6e01515661591a41320a63451cb82d5f;p=clo diff --git a/src/index.ts b/src/index.ts index 0d94c74..405a1f0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,114 +1,10 @@ var fs = require('fs'); -import { argv, resourceUsage } from 'node:process'; -import * as tk from './tokenize.js'; +import jsTokens from "js-tokens"; import * as util from 'util'; -import { drawEllipsePath, reduceRotation } from 'pdf-lib'; -import { isTypedArray } from 'node:util/types'; -import { error } from 'node:console'; -/** - * debug reprensenting - */ +/**inspect the inner of the representation. */ let repr = (x : any)=>{return util.inspect(x, {depth: null})}; -/** - * token tree type. - */ -type tkTree = tkTree[] | tk.Token - -/** - * concated 2 `tkTree`s - * @param x the array to be concated - * @param y the item or array to ve concated - * @returns concated tkTree array, or thrown error if can't be concated. - */ -function concat(x: tkTree, y:tkTree): tkTree[] { - if (Array.isArray(x)){ - return x.concat(y); - }else{ - throw new Error("the tkTree can't be concated, because it's not an array."); - - } -} - -function slice(x: tkTree, index?:number, end?:number): tkTree[] { - if (Array.isArray(x)){ - return x.slice(index,end); - }else{ - throw new Error("the tkTree can't be concated, because it's not an array."); - - } -} - -/** - * TokenMatcheePair for tokens' parser combinator - * matched: the matched (now and before) tokens - * remained: tokens to be matched - * ast: abstract syntax tree - */ -export interface TokenMatcheePair { - matched: tk.Token[] - remained: tk.Token[] - ast : tkTree[] -} - -/** - * @description - * match one token type. - * - * it returns a function which test if the type of first token of the `remained` part of - * the argument of the function is `typ` , if it's true, update the `TokenMatcheePair` wrapped - * in `Some`. Otherwise, it returns `None`. - * * @param typ : the type to be test. - * @returns the updated `TokenMatcheePair` wrapped in `Some(x)` or `None`. - */ -export function m1TType(typ: tk.TokenType): - (m: TokenMatcheePair) => tk.Maybe { - return (m: TokenMatcheePair) => { - if (m.remained.length == 0) { - return { _tag: "None" }; - } - /** - * token to be matched - * */ - const ttbm = m.remained[0]; - - if (ttbm.type == typ) { - let new_matched = m.matched.concat(ttbm); - let result : tk.Some = { - _tag: "Some", value: { - matched: new_matched, - remained: m.remained.slice(1), - ast: ([ttbm]), - } - }; - return result; - } - else { - return { _tag: "None" }; - } - } -}; - -/** - * type int - */ -let tInt = m1TType(tk.TokenType.INT); -let tAdd = m1TType(tk.TokenType.I_ADD); -let tSub = m1TType(tk.TokenType.I_SUB); -let tMul = m1TType(tk.TokenType.I_MUL); -let tDiv = m1TType(tk.TokenType.I_DIV); -let tLParen = m1TType(tk.TokenType.L_PAREN); -let tRParen = m1TType(tk.TokenType.R_PAREN); - -let toSome = tk.toSome; - - -argv.forEach((val, index) => { - console.log(`${index}=${val}`); -}); - - /** * like `m ==> f` in ocaml * @param m matchee wrapped @@ -128,137 +24,14 @@ function thenDo(m : tk.Maybe, f : Function){ } } -/** - * like `f1 | f2` in regex - * @param f1 the first tried function - * @param f2 the second tried function - * @returns wrapped result - */ -function orDo(f1 : Function, f2 : Function){ - return (x : TokenMatcheePair) =>{ - let res1 : tk.Maybe = f1(x); - if (res1._tag == "Some"){ - return res1; - }else{ - let res2 : tk.Maybe = f2(x); - return res2; - } - } - -} - -/** - * aux function for midfix operator - * @param f function - * @param signal the rule name - * @returns - */ -let midfix = (f : Function, signal? : string) => (x : TokenMatcheePair)=>{ - var a = f(x); - if (a._tag == "Some"){ - let ast_tail : tkTree[] = slice(a.value.ast,a.value.ast.length-3); - let new_ast = [ast_tail]; - a.value.ast = new_ast; - - console.log("+"+signal+"+"+repr(a)); - - - } - return a; -} - -let circumfix = (f : Function, signal? : string) => (x : TokenMatcheePair)=>{ - var a = f(x); - if (a._tag == "Some"){ - let inner = a.value.ast[a.value.ast.length-2]; - let ast_middle : tkTree[] = [inner]; - let new_ast = [ast_middle]; - a.value.ast = new_ast; - } - return a; -} - -/** - * TODO: 12(13)(14) only parsed with only 12(13) - */ -/** single1 = tInt | "(" expr ")"*/ -let single1 = circumfix((x : TokenMatcheePair) => - thenDo(thenDo(thenDo(tk.toSome(x), tLParen), expr), tRParen), "fac1"); -let single2= tInt; -let single = orDo(single1, single2); - -/** fac1 = single "(" int ")" | single */ -let fac1Appliee = circumfix((x : TokenMatcheePair) => thenDo(thenDo(thenDo(tk.toSome(x), tLParen), tInt), tRParen), "fac1"); -let fac1 = (x : TokenMatcheePair) => - { - let n = thenDo(thenDo(toSome(x), single), fac1Appliee); - - console.log("+"+"bocchitherock"+"+"+repr(n)); - - return n - }; -let fac2 = single; -let fac = orDo(fac1, fac2); - - -/** - * - * term1 = fac (MUL | DIV) fac - */ - -let term1 = midfix((x : TokenMatcheePair)=> - thenDo(thenDo(thenDo(tk.toSome(x), fac), orDo(tMul,tDiv)), fac), "term1"); - - -/** - * - * term2 = int MUL int - */ -let term2 = fac; - -/** - * term = term1 | term2 - */ -let term = orDo(term1, term2); - - -/** - * - * expr1 = term ADD term - */ -let expr1 = midfix((x : TokenMatcheePair)=> - thenDo(thenDo(thenDo(tk.toSome(x), term), orDo(tAdd,tSub)), term), "expr1"); -/** - * expr2 = term - */ -let expr2 = term; - -/** - * expr = expr1 | expr2 - */ -let expr = orDo(expr1, expr2); - - - -let tokens = tk.tokenize("12(13)(14)"); -//let tokens = tk.tokenize("(4-(3/4))"); -//tk.tokenize(argv[2]); - -let tokensFiltered = tokens.filter( - (x)=>{return (x.type != tk.TokenType.NL - && x.type != tk.TokenType.SP)}); +const tokens = Array.from(jsTokens(` +import foo from\t 'bar'; +import * as util from 'util'; -let wrappedTokens : tk.Maybe = - tk.toSome({ - matched : [] , - remained : tokensFiltered, - ast : []}); -let beta = expr({ - matched : [] , - remained : tokensFiltered, - ast : []}); +花非花，霧\\{非霧 。{{foo();}}下 +一句`)); -console.log("RESULT="+repr(beta)); +console.log("RESULT="+repr(tokens));