add funtions of `tokenizer`

[clo] / tests / index.js

diff --git a/tests/index.js b/tests/index.js
index 891511cc0da8b6c121cfca83346032f3d20a378f..36eefccb13e5501e5112f3630a855ffc9b957bb9 100644 (file)
--- a/tests/index.js
+++ b/tests/index.js
@@ -57,6 +57,35 @@ let doTestRes9 = thenDo(doThenTestee9, src_1.matchAny);
 assert(doTestRes9._tag == "Some");
 assert(doTestRes9.value.matched == "妳");
 assert(doTestRes9.value.remained == "的");
+(0, src_1.tokenize)("+123");
+(0, src_1.tokenize)("123");
+(0, src_1.tokenize)("-123");
+(0, src_1.tokenize)(" 123");
+try {
+    (0, src_1.tokenize)("c123");
+}
+catch (error) {
+    console.log(error);
+}
+(0, src_1.tokenize)("  ");
+(0, src_1.tokenize)(" ");
+(0, src_1.tokenize)(" \t");
+(0, src_1.tokenize)(" \t123");
+try {
+    (0, src_1.tokenize)(" \t123aaa456");
+}
+catch (error) {
+    console.log(error);
+}
+(0, src_1.tokenize)(" \t123\n456");
+(0, src_1.tokenize)("\"\"");
+(0, src_1.tokenize)("\"123\"");
+(0, src_1.tokenize)("\"1\\\"23\"");
+(0, src_1.tokenize)("\"1\\\"23\"  abc123");
+(0, src_1.tokenize)("+0.012");
+(0, src_1.tokenize)("0.0");
+(0, src_1.tokenize)("-222.0");
+(0, src_1.tokenize)("1+1 ==2; 3+8 foo(12)");
 // harfbuzz test
 let harfbuzz = require("../src/harfbuzz.js");
 harfbuzz.harfbuzzTest("123.abc");