languages/cpython
languages/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython synced 2024-09-04 15:56:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

bpo-39209: Manage correctly multi-line tokens in interactive mode (GH-17860)

Browse source
This commit is contained in:
Pablo Galindo 2020-01-06 15:59:09 +00:00 committed by GitHub
parent 075ebad369
commit 5ec91f78d5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 40 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
Lib/test/test_repl.py
View file

@ -58,5 +58,41 @@ def test_no_memory(self):
# Exit code 120: Py_FinalizeEx() failed to flush stdout and stderr.
self.assertIn(p.returncode, (1, 120))
@cpython_only
def test_multiline_string_parsing(self):
# bpo-39209: Multiline string tokens need to be handled in the tokenizer
# in two places: the interactive path and the non-interactive path.
user_input = '''\
x = """<?xml version="1.0" encoding="iso-8859-1"?>
<test>
<Users>
<fun25>
<limits>
<total>0KiB</total>
<kbps>0</kbps>
<rps>1.3</rps>
<connections>0</connections>
</limits>
<usages>
<total>16738211KiB</total>
<kbps>237.15</kbps>
<rps>1.3</rps>
<connections>0</connections>
</usages>
<time_to_refresh>never</time_to_refresh>
<limit_exceeded_URL>none</limit_exceeded_URL>
</fun25>
</Users>
</test>"""
'''
user_input = dedent(user_input)
user_input = user_input.encode()
p = spawn_repl()
with SuppressCrashReport():
p.stdin.write(user_input)
output = kill_python(p)
self.assertEqual(p.returncode, 0)
if __name__ == "__main__":
unittest.main()

2
Misc/NEWS.d/next/Core and Builtins/2020-01-06-10-29-16.bpo-39209.QHAONe.rst Normal file
View file

@ -0,0 +1,2 @@
Correctly handle multi-line tokens in interactive mode. Patch by Pablo
Galindo.

2
Parser/tokenizer.c
View file

@ -886,6 +886,7 @@ tok_nextc(struct tok_state *tok)
size_t start = tok->start - tok->buf;
size_t oldlen = tok->cur - tok->buf;
size_t newlen = oldlen + strlen(newtok);
Py_ssize_t cur_multi_line_start = tok->multi_line_start - tok->buf;
char *buf = tok->buf;
buf = (char *)PyMem_REALLOC(buf, newlen+1);
tok->lineno++;
@ -898,6 +899,7 @@ tok_nextc(struct tok_state *tok)
}
tok->buf = buf;
tok->cur = tok->buf + oldlen;
tok->multi_line_start = tok->buf + cur_multi_line_start;
tok->line_start = tok->cur;
strcpy(tok->buf + oldlen, newtok);
PyMem_FREE(newtok);