Function tokenize

json/token_test.go:44–72

(t *testing.T, b []byte)

}

func tokenize(t *testing.T, b []byte) (tokens []token) {
	tok := NewTokenizer(b)

	for tok.Next() {
		end := len(b) - tok.Remaining()
		start := end - len(tok.Value)
		if end > len(b) {
			t.Fatalf("token position too far [%d:%d], len(b) is %d", start, end, len(b))
		}
		if !bytes.Equal(b[start:end], tok.Value) {
			t.Fatalf("token position is wrong [%d:%d]", start, end)
		}

		tokens = append(tokens, token{
			delim: tok.Delim,
			value: tok.Value,
			err:   tok.Err,
			depth: tok.Depth,
			index: tok.Index,
			isKey: tok.IsKey,
		})
	}

	if tok.Err != nil {
		t.Fatal(tok.Err)
	}

	return
}

func TestTokenizer(t *testing.T) {
	tests := []struct {