diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index e34bd7b83a19b0..f8d06b469b651e 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -230,6 +230,23 @@ def test_error_offset_continuation_characters(self): check = self.check check('"\\\n"(1 for c in I,\\\n\\', 2, 2) + def testSyntaxErrorRange(self): + # gh-156894: the position was reported in bytes, not in characters, + # for the errors which cover a range + for source, offset, end_offset in [ + ('abcd = 00010', 8, 11), + ('\u03b1\u03b2\u03b3\u03b4 = 00010', 8, 11), + ('a\u0301b\u0308c\u20d7d\u1ab0 = 00010', 12, 15), + ("abcd = ub'a'", 10, 13), + ("\u03b1\u03b2\u03b3\u03b4 = ub'a'", 10, 13), + ("a\u0301b\u0308c\u20d7d\u1ab0 = ub'a'", 14, 17), + ]: + with self.subTest(source=source): + with self.assertRaises(SyntaxError) as cm: + compile(source, '', 'exec') + self.assertEqual(cm.exception.offset, offset) + self.assertEqual(cm.exception.end_offset, end_offset) + def testSyntaxErrorOffset(self): check = self.check check('def fact(x):\n\treturn x!\n', 2, 10) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-03-15-20-00.gh-issue-156894.Rt9Bx4.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-03-15-20-00.gh-issue-156894.Rt9Bx4.rst new file mode 100644 index 00000000000000..9b279ab2794786 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-03-15-20-00.gh-issue-156894.Rt9Bx4.rst @@ -0,0 +1,2 @@ +Fix the position of syntax errors which cover a range if the line contains +non-ASCII characters before the error. diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index 976d039f571554..aad30b90cbf879 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -7,6 +7,20 @@ /* ############## ERRORS ############## */ +/* Convert a 1-based column in bytes into a 1-based column in characters. + The line is UTF-8 encoded, so it is enough to skip continuation bytes. */ +static int +byte_col_to_char_col(const char *line, int byte_col) +{ + int char_col = 1; + for (int i = 0; i < byte_col - 1; i++) { + if ((line[i] & 0xC0) != 0x80) { + char_col++; + } + } + return char_col; +} + static int _syntaxerror_range(struct tok_state *tok, const char *format, int col_offset, int end_col_offset, @@ -33,9 +47,15 @@ _syntaxerror_range(struct tok_state *tok, const char *format, if (col_offset == -1) { col_offset = (int)PyUnicode_GET_LENGTH(errtext); } + else if (col_offset > 0) { + col_offset = byte_col_to_char_col(tok->line_start, col_offset); + } if (end_col_offset == -1) { end_col_offset = col_offset; } + else if (end_col_offset > 0) { + end_col_offset = byte_col_to_char_col(tok->line_start, end_col_offset); + } Py_ssize_t line_len = strcspn(tok->line_start, "\n"); if (line_len != tok->cur - tok->line_start) {