add tokenize's function, add interface `Token`

[clo] / tests / index.js

diff --git a/tests/index.js b/tests/index.js
index 891511cc0da8b6c121cfca83346032f3d20a378f..b50c10341a49282307208bb1cee07f797eccf462 100644 (file)
--- a/tests/index.js
+++ b/tests/index.js
@@ -57,6 +57,27 @@ let doTestRes9 = thenDo(doThenTestee9, src_1.matchAny);
 assert(doTestRes9._tag == "Some");
 assert(doTestRes9.value.matched == "妳");
 assert(doTestRes9.value.remained == "的");
+(0, src_1.tokenize)("+123");
+(0, src_1.tokenize)("123");
+(0, src_1.tokenize)("-123");
+(0, src_1.tokenize)(" 123");
+try {
+    (0, src_1.tokenize)("c123");
+}
+catch (error) {
+    console.log(error);
+}
+(0, src_1.tokenize)("  ");
+(0, src_1.tokenize)(" ");
+(0, src_1.tokenize)(" \t");
+(0, src_1.tokenize)(" \t123");
+try {
+    (0, src_1.tokenize)(" \t123aaa456");
+}
+catch (error) {
+    console.log(error);
+}
+(0, src_1.tokenize)(" \t123\n456");
 // harfbuzz test
 let harfbuzz = require("../src/harfbuzz.js");
 harfbuzz.harfbuzzTest("123.abc");