Fix a crash bug

mnogu · mnogu · commit 975827a9d768 · 2020-10-25T20:00:39.000+09:00
diff --git a/tokenizer.go b/tokenizer.go
@@ -13,6 +13,7 @@ const (
 	reservedToken tokenKind = "reserved"
 	numberToken   tokenKind = "number"
 	identToken    tokenKind = "ident"
+	eosToken      tokenKind = "eos"
 )
 
 type token struct {
@@ -113,5 +114,6 @@ func tokenize(input string) ([]token, error) {
 
 		return nil, &invalidTokenError{input: input, position: i}
 	}
+	tokens = append(tokens, token{kind: eosToken})
 	return tokens, nil
 }

Original file line number	Diff line number	Diff line change
`@@ -13,6 +13,7 @@ const (`
`13`	`13`	`reservedToken tokenKind = "reserved"`
`14`	`14`	`numberToken tokenKind = "number"`
`15`	`15`	`identToken tokenKind = "ident"`
	`16`	`+ eosToken tokenKind = "eos"`
`16`	`17`	`)`
`17`	`18`
`18`	`19`	`type token struct {`
`@@ -113,5 +114,6 @@ func tokenize(input string) ([]token, error) {`
`113`	`114`
`114`	`115`	`return nil, &invalidTokenError{input: input, position: i}`
`115`	`116`	`}`
	`117`	`+ tokens = append(tokens, token{kind: eosToken})`
`116`	`118`	`return tokens, nil`
`117`	`119`	`}`