add funtions of `tokenizer`

[clo] / tests / index.js

diff --git a/tests/index.js b/tests/index.js
index b50c10341a49282307208bb1cee07f797eccf462..36eefccb13e5501e5112f3630a855ffc9b957bb9 100644 (file)
--- a/tests/index.js
+++ b/tests/index.js
@@ -78,6 +78,14 @@ catch (error) {
     console.log(error);
 }
 (0, src_1.tokenize)(" \t123\n456");
+(0, src_1.tokenize)("\"\"");
+(0, src_1.tokenize)("\"123\"");
+(0, src_1.tokenize)("\"1\\\"23\"");
+(0, src_1.tokenize)("\"1\\\"23\"  abc123");
+(0, src_1.tokenize)("+0.012");
+(0, src_1.tokenize)("0.0");
+(0, src_1.tokenize)("-222.0");
+(0, src_1.tokenize)("1+1 ==2; 3+8 foo(12)");
 // harfbuzz test
 let harfbuzz = require("../src/harfbuzz.js");
 harfbuzz.harfbuzzTest("123.abc");