Skip to content

Commit 5fe5697

Browse files
alisdairalisdair
committed
Meld consecutive template string literals
When processing a template string, the lexer can emit multiple string literal tokens for what ought to be a single string literal. This occurs when the string contains escape sequences, or consecutive characters which are indistinguishable from escape sequences at tokenization time. This leads to a confusing AST and causes heuristics about template expressions to fail. Specifically, when parsing a traversal with an index, a key value containing an escape symbol will cause the parser to generate an index expression instead of a traversal. This commit adds a post-processing step to the template parser to meld any sequences of string literals into a single string literal. Existing tests covered the previous misbehaviour (several of which had comments apologizing for it), and have been updated accordingly. The new behaviour of the `IsStringLiteral` method of `TemplateExpr` is covered with a new set of tests.
1 parent 67270ba commit 5fe5697

File tree

3 files changed

+79
-92
lines changed

3 files changed

+79
-92
lines changed

‎hclsyntax/expression_template_test.go

+41
Original file line numberDiff line numberDiff line change
@@ -395,3 +395,44 @@ trim`,
395395
}
396396

397397
}
398+
399+
func TestTemplateExprIsStringLiteral(t *testing.T) {
400+
tests := map[string]bool{
401+
// A simple string value is a string literal
402+
"a": true,
403+
404+
// Strings containing escape characters or escape sequences are
405+
// tokenized into multiple string literals, but this should be
406+
// corrected by the parser
407+
"a$b": true,
408+
"a%%b": true,
409+
"a\nb": true,
410+
"a$${\"b\"}": true,
411+
412+
// Wrapped values (HIL-like) are not treated as string literals for
413+
// legacy reasons
414+
"${1}": false,
415+
"${\"b\"}": false,
416+
417+
// Even template expressions containing only literal values do not
418+
// count as string literals
419+
"a${1}": false,
420+
"a${\"b\"}": false,
421+
}
422+
for input, want := range tests {
423+
t.Run(input, func(t *testing.T) {
424+
expr, diags := ParseTemplate([]byte(input), "", hcl.InitialPos)
425+
if len(diags) != 0 {
426+
t.Fatalf("unexpected diags: %s", diags.Error())
427+
}
428+
429+
if tmplExpr, ok := expr.(*TemplateExpr); ok {
430+
got := tmplExpr.IsStringLiteral()
431+
432+
if got != want {
433+
t.Errorf("wrong result\ngot: %#v\nwant: %#v", got, want)
434+
}
435+
}
436+
})
437+
}
438+
}

‎hclsyntax/parser_template.go

+32
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ func (p *parser) parseTemplateInner(end TokenType, flushHeredoc bool) ([]Express
3838
if flushHeredoc {
3939
flushHeredocTemplateParts(parts) // Trim off leading spaces on lines per the flush heredoc spec
4040
}
41+
meldConsecutiveStringLiterals(parts)
4142
tp := templateParser{
4243
Tokens: parts.Tokens,
4344
SrcRange: parts.SrcRange,
@@ -751,6 +752,37 @@ func flushHeredocTemplateParts(parts *templateParts) {
751752
}
752753
}
753754

755+
// meldConsecutiveStringLiterals simplifies the AST output by combining a
756+
// sequence of string literal tokens into a single string literal. This must be
757+
// performed after any whitespace trimming operations.
758+
func meldConsecutiveStringLiterals(parts *templateParts) {
759+
if len(parts.Tokens) == 0 {
760+
return
761+
}
762+
763+
// Loop over all tokens starting at the second element, as we want to join
764+
// pairs of consecutive string literals.
765+
i := 1
766+
for i < len(parts.Tokens) {
767+
if prevLiteral, ok := parts.Tokens[i-1].(*templateLiteralToken); ok {
768+
if literal, ok := parts.Tokens[i].(*templateLiteralToken); ok {
769+
// The current and previous tokens are both literals: combine
770+
prevLiteral.Val = prevLiteral.Val + literal.Val
771+
prevLiteral.SrcRange.End = literal.SrcRange.End
772+
773+
// Remove the current token from the slice
774+
parts.Tokens = append(parts.Tokens[:i], parts.Tokens[i+1:]...)
775+
776+
// Continue without moving forward in the slice
777+
continue
778+
}
779+
}
780+
781+
// Try the next pair of tokens
782+
i++
783+
}
784+
}
785+
754786
type templateParts struct {
755787
Tokens []templateToken
756788
SrcRange hcl.Range

‎hclsyntax/parser_test.go

+6-92
Original file line numberDiff line numberDiff line change
@@ -738,26 +738,10 @@ block "valid" {}
738738
Expr: &TemplateExpr{
739739
Parts: []Expression{
740740
&LiteralValueExpr{
741-
Val: cty.StringVal("hello "),
741+
Val: cty.StringVal("hello ${true}"),
742742

743743
SrcRange: hcl.Range{
744744
Start: hcl.Pos{Line: 1, Column: 6, Byte: 5},
745-
End: hcl.Pos{Line: 1, Column: 12, Byte: 11},
746-
},
747-
},
748-
&LiteralValueExpr{
749-
Val: cty.StringVal("${"),
750-
751-
SrcRange: hcl.Range{
752-
Start: hcl.Pos{Line: 1, Column: 12, Byte: 11},
753-
End: hcl.Pos{Line: 1, Column: 15, Byte: 14},
754-
},
755-
},
756-
&LiteralValueExpr{
757-
Val: cty.StringVal("true}"),
758-
759-
SrcRange: hcl.Range{
760-
Start: hcl.Pos{Line: 1, Column: 15, Byte: 14},
761745
End: hcl.Pos{Line: 1, Column: 20, Byte: 19},
762746
},
763747
},
@@ -804,26 +788,10 @@ block "valid" {}
804788
Expr: &TemplateExpr{
805789
Parts: []Expression{
806790
&LiteralValueExpr{
807-
Val: cty.StringVal("hello "),
791+
Val: cty.StringVal("hello %{true}"),
808792

809793
SrcRange: hcl.Range{
810794
Start: hcl.Pos{Line: 1, Column: 6, Byte: 5},
811-
End: hcl.Pos{Line: 1, Column: 12, Byte: 11},
812-
},
813-
},
814-
&LiteralValueExpr{
815-
Val: cty.StringVal("%{"),
816-
817-
SrcRange: hcl.Range{
818-
Start: hcl.Pos{Line: 1, Column: 12, Byte: 11},
819-
End: hcl.Pos{Line: 1, Column: 15, Byte: 14},
820-
},
821-
},
822-
&LiteralValueExpr{
823-
Val: cty.StringVal("true}"),
824-
825-
SrcRange: hcl.Range{
826-
Start: hcl.Pos{Line: 1, Column: 15, Byte: 14},
827795
End: hcl.Pos{Line: 1, Column: 20, Byte: 19},
828796
},
829797
},
@@ -870,29 +838,10 @@ block "valid" {}
870838
Expr: &TemplateExpr{
871839
Parts: []Expression{
872840
&LiteralValueExpr{
873-
Val: cty.StringVal("hello "),
841+
Val: cty.StringVal("hello $$"),
874842

875843
SrcRange: hcl.Range{
876844
Start: hcl.Pos{Line: 1, Column: 6, Byte: 5},
877-
End: hcl.Pos{Line: 1, Column: 12, Byte: 11},
878-
},
879-
},
880-
// This parses oddly due to how the scanner
881-
// handles escaping of the $ sequence, but it's
882-
// functionally equivalent to a single literal.
883-
&LiteralValueExpr{
884-
Val: cty.StringVal("$"),
885-
886-
SrcRange: hcl.Range{
887-
Start: hcl.Pos{Line: 1, Column: 12, Byte: 11},
888-
End: hcl.Pos{Line: 1, Column: 13, Byte: 12},
889-
},
890-
},
891-
&LiteralValueExpr{
892-
Val: cty.StringVal("$"),
893-
894-
SrcRange: hcl.Range{
895-
Start: hcl.Pos{Line: 1, Column: 13, Byte: 12},
896845
End: hcl.Pos{Line: 1, Column: 14, Byte: 13},
897846
},
898847
},
@@ -939,18 +888,10 @@ block "valid" {}
939888
Expr: &TemplateExpr{
940889
Parts: []Expression{
941890
&LiteralValueExpr{
942-
Val: cty.StringVal("hello "),
891+
Val: cty.StringVal("hello $"),
943892

944893
SrcRange: hcl.Range{
945894
Start: hcl.Pos{Line: 1, Column: 6, Byte: 5},
946-
End: hcl.Pos{Line: 1, Column: 12, Byte: 11},
947-
},
948-
},
949-
&LiteralValueExpr{
950-
Val: cty.StringVal("$"),
951-
952-
SrcRange: hcl.Range{
953-
Start: hcl.Pos{Line: 1, Column: 12, Byte: 11},
954895
End: hcl.Pos{Line: 1, Column: 13, Byte: 12},
955896
},
956897
},
@@ -997,29 +938,10 @@ block "valid" {}
997938
Expr: &TemplateExpr{
998939
Parts: []Expression{
999940
&LiteralValueExpr{
1000-
Val: cty.StringVal("hello "),
941+
Val: cty.StringVal("hello %%"),
1001942

1002943
SrcRange: hcl.Range{
1003944
Start: hcl.Pos{Line: 1, Column: 6, Byte: 5},
1004-
End: hcl.Pos{Line: 1, Column: 12, Byte: 11},
1005-
},
1006-
},
1007-
// This parses oddly due to how the scanner
1008-
// handles escaping of the % sequence, but it's
1009-
// functionally equivalent to a single literal.
1010-
&LiteralValueExpr{
1011-
Val: cty.StringVal("%"),
1012-
1013-
SrcRange: hcl.Range{
1014-
Start: hcl.Pos{Line: 1, Column: 12, Byte: 11},
1015-
End: hcl.Pos{Line: 1, Column: 13, Byte: 12},
1016-
},
1017-
},
1018-
&LiteralValueExpr{
1019-
Val: cty.StringVal("%"),
1020-
1021-
SrcRange: hcl.Range{
1022-
Start: hcl.Pos{Line: 1, Column: 13, Byte: 12},
1023945
End: hcl.Pos{Line: 1, Column: 14, Byte: 13},
1024946
},
1025947
},
@@ -1066,18 +988,10 @@ block "valid" {}
1066988
Expr: &TemplateExpr{
1067989
Parts: []Expression{
1068990
&LiteralValueExpr{
1069-
Val: cty.StringVal("hello "),
991+
Val: cty.StringVal("hello %"),
1070992

1071993
SrcRange: hcl.Range{
1072994
Start: hcl.Pos{Line: 1, Column: 6, Byte: 5},
1073-
End: hcl.Pos{Line: 1, Column: 12, Byte: 11},
1074-
},
1075-
},
1076-
&LiteralValueExpr{
1077-
Val: cty.StringVal("%"),
1078-
1079-
SrcRange: hcl.Range{
1080-
Start: hcl.Pos{Line: 1, Column: 12, Byte: 11},
1081995
End: hcl.Pos{Line: 1, Column: 13, Byte: 12},
1082996
},
1083997
},

0 commit comments

Comments
 (0)