add funtions of `tokenizer`

[clo] / tests / index.ts

diff --git a/tests/index.ts b/tests/index.ts
index e4c7344d16b9bae8700ee7c4fbf6bc6d63436c13..e6465dcca241b7c978897a74eec95ee34280f198 100644 (file)
--- a/tests/index.ts
+++ b/tests/index.ts
@@ -99,6 +99,15 @@ try {
 }
 tokenize(" \t123\n456");
 
+tokenize("\"\"");
+tokenize("\"123\"");
+tokenize("\"1\\\"23\"");
+tokenize("\"1\\\"23\"  abc123");
+
+tokenize("+0.012");
+tokenize("0.0");
+tokenize("-222.0");
+tokenize("1+1 ==2; 3+8 foo(12)");
 
 // harfbuzz test
 let harfbuzz = require("../src/harfbuzz.js");