Skip to content

Commit 70f315c

Browse files
authored
gh-105042: Disable unmatched parens syntax error in python tokenize (#105061)
1 parent 9216e69 commit 70f315c

File tree

5 files changed

+49
-34
lines changed

5 files changed

+49
-34
lines changed

Lib/test/inspect_fodder.py

+5
Original file line numberDiff line numberDiff line change
@@ -113,3 +113,8 @@ async def asyncf(self):
113113
# after asyncf - line 113
114114
# end of WhichComments - line 114
115115
# after WhichComments - line 115
116+
117+
# Test that getsource works on a line that includes
118+
# a closing parenthesis with the opening paren being in another line
119+
(
120+
); after_closing = lambda: 1

Lib/test/test_inspect.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -557,7 +557,8 @@ def test_getclasses(self):
557557

558558
def test_getfunctions(self):
559559
functions = inspect.getmembers(mod, inspect.isfunction)
560-
self.assertEqual(functions, [('eggs', mod.eggs),
560+
self.assertEqual(functions, [('after_closing', mod.after_closing),
561+
('eggs', mod.eggs),
561562
('lobbest', mod.lobbest),
562563
('spam', mod.spam)])
563564

@@ -641,6 +642,7 @@ def test_getsource(self):
641642
self.assertSourceEqual(git.abuse, 29, 39)
642643
self.assertSourceEqual(mod.StupidGit, 21, 51)
643644
self.assertSourceEqual(mod.lobbest, 75, 76)
645+
self.assertSourceEqual(mod.after_closing, 120, 120)
644646

645647
def test_getsourcefile(self):
646648
self.assertEqual(normcase(inspect.getsourcefile(mod.spam)), modfile)

Lib/test/test_tokenize.py

+7
Original file line numberDiff line numberDiff line change
@@ -1119,6 +1119,13 @@ def test_newline_after_parenthesized_block_with_comment(self):
11191119
NEWLINE '\\n' (4, 1) (4, 2)
11201120
""")
11211121

1122+
def test_closing_parenthesis_from_different_line(self):
1123+
self.check_tokenize("); x", """\
1124+
OP ')' (1, 0) (1, 1)
1125+
OP ';' (1, 1) (1, 2)
1126+
NAME 'x' (1, 3) (1, 4)
1127+
""")
1128+
11221129
class GenerateTokensTest(TokenizeTest):
11231130
def check_tokenize(self, s, expected):
11241131
# Format the tokens in s in a table format.

Parser/tokenizer.c

+33-32
Original file line numberDiff line numberDiff line change
@@ -2626,41 +2626,42 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
26262626
case ')':
26272627
case ']':
26282628
case '}':
2629-
if (!tok->level) {
2630-
if (INSIDE_FSTRING(tok) && !current_tok->curly_bracket_depth && c == '}') {
2631-
return MAKE_TOKEN(syntaxerror(tok, "f-string: single '}' is not allowed"));
2632-
}
2629+
if (INSIDE_FSTRING(tok) && !current_tok->curly_bracket_depth && c == '}') {
2630+
return MAKE_TOKEN(syntaxerror(tok, "f-string: single '}' is not allowed"));
2631+
}
2632+
if (!tok->tok_extra_tokens && !tok->level) {
26332633
return MAKE_TOKEN(syntaxerror(tok, "unmatched '%c'", c));
26342634
}
2635-
tok->level--;
2636-
int opening = tok->parenstack[tok->level];
2637-
if (!((opening == '(' && c == ')') ||
2638-
(opening == '[' && c == ']') ||
2639-
(opening == '{' && c == '}')))
2640-
{
2641-
/* If the opening bracket belongs to an f-string's expression
2642-
part (e.g. f"{)}") and the closing bracket is an arbitrary
2643-
nested expression, then instead of matching a different
2644-
syntactical construct with it; we'll throw an unmatched
2645-
parentheses error. */
2646-
if (INSIDE_FSTRING(tok) && opening == '{') {
2647-
assert(current_tok->curly_bracket_depth >= 0);
2648-
int previous_bracket = current_tok->curly_bracket_depth - 1;
2649-
if (previous_bracket == current_tok->curly_bracket_expr_start_depth) {
2650-
return MAKE_TOKEN(syntaxerror(tok, "f-string: unmatched '%c'", c));
2635+
if (tok->level > 0) {
2636+
tok->level--;
2637+
int opening = tok->parenstack[tok->level];
2638+
if (!tok->tok_extra_tokens && !((opening == '(' && c == ')') ||
2639+
(opening == '[' && c == ']') ||
2640+
(opening == '{' && c == '}'))) {
2641+
/* If the opening bracket belongs to an f-string's expression
2642+
part (e.g. f"{)}") and the closing bracket is an arbitrary
2643+
nested expression, then instead of matching a different
2644+
syntactical construct with it; we'll throw an unmatched
2645+
parentheses error. */
2646+
if (INSIDE_FSTRING(tok) && opening == '{') {
2647+
assert(current_tok->curly_bracket_depth >= 0);
2648+
int previous_bracket = current_tok->curly_bracket_depth - 1;
2649+
if (previous_bracket == current_tok->curly_bracket_expr_start_depth) {
2650+
return MAKE_TOKEN(syntaxerror(tok, "f-string: unmatched '%c'", c));
2651+
}
2652+
}
2653+
if (tok->parenlinenostack[tok->level] != tok->lineno) {
2654+
return MAKE_TOKEN(syntaxerror(tok,
2655+
"closing parenthesis '%c' does not match "
2656+
"opening parenthesis '%c' on line %d",
2657+
c, opening, tok->parenlinenostack[tok->level]));
2658+
}
2659+
else {
2660+
return MAKE_TOKEN(syntaxerror(tok,
2661+
"closing parenthesis '%c' does not match "
2662+
"opening parenthesis '%c'",
2663+
c, opening));
26512664
}
2652-
}
2653-
if (tok->parenlinenostack[tok->level] != tok->lineno) {
2654-
return MAKE_TOKEN(syntaxerror(tok,
2655-
"closing parenthesis '%c' does not match "
2656-
"opening parenthesis '%c' on line %d",
2657-
c, opening, tok->parenlinenostack[tok->level]));
2658-
}
2659-
else {
2660-
return MAKE_TOKEN(syntaxerror(tok,
2661-
"closing parenthesis '%c' does not match "
2662-
"opening parenthesis '%c'",
2663-
c, opening));
26642665
}
26652666
}
26662667

Python/Python-tokenize.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ _tokenizer_error(struct tok_state *tok)
8484
msg = "invalid token";
8585
break;
8686
case E_EOF:
87-
if (tok->level) {
87+
if (tok->level > 0) {
8888
PyErr_Format(PyExc_SyntaxError,
8989
"parenthesis '%c' was never closed",
9090
tok->parenstack[tok->level-1]);

0 commit comments

Comments
 (0)