Clean up tokenizer implementation

Rather than storing a bunch of "next_foo" fields, simply populate the
tok_t directly.
This commit is contained in:
ridiculousfish
2018-02-23 14:30:15 -08:00
parent e9a4875a6b
commit 6673fe5457
7 changed files with 122 additions and 130 deletions

View File

@@ -519,14 +519,14 @@ static void test_tokenizer() {
do_test(token.type == TOK_STRING);
do_test(token.offset == 0);
do_test(token.length == 5);
do_test(token.text == L"alpha");
do_test(t.text_of(token) == L"alpha");
got = t.next(&token); // beta
do_test(got);
do_test(token.type == TOK_STRING);
do_test(token.offset == 6);
do_test(token.length == 4);
do_test(token.text == L"beta");
do_test(t.text_of(token) == L"beta");
got = t.next(&token);
do_test(!got);