From 1c60884e949310002be310ee17c6220918ae450e Mon Sep 17 00:00:00 2001 From: Brittany Reynoso Date: Wed, 2 Sep 2026 14:12:23 -0700 Subject: [PATCH] gh-156955: Speed up csv.writer by not scanning lineterminator per character join_append_data() tested every character of every field for membership in dialect->lineterminator with PyUnicode_FindChar(), an out-of-line call made twice per character (the function runs a count pass and a copy pass). At 54% of cycles it was the single hottest symbol in csv.writer, which made writing CSV slower than parsing it back. Cache the terminator's highest code point on the dialect, which is immutable, and compare inline. Ordinary text exceeds that maximum, so the membership test is skipped without touching the terminator at all; when it does run it is a short loop over the terminator's characters rather than a cross-module call. Membership semantics are unchanged, including multi-character, empty and non-BMP terminators. Interleaved A/B, median of 25 per-round ratios, pinned to one CPU: mixed 2000x4 1.430 ms -> 0.547 ms 2.62x text 2000x4 1.486 ms -> 0.561 ms 2.64x wide 500x2 (200ch) 3.159 ms -> 1.094 ms 2.88x mixed QUOTE_ALL 1.438 ms -> 0.606 ms 2.37x mixed lineterm='\n' 1.335 ms -> 0.605 ms 2.21x mixed lineterm='END' 1.535 ms -> 0.822 ms 1.87x quoted 2000x4 0.486 ms -> 0.323 ms 1.50x short 5000x4 0.906 ms -> 0.650 ms 1.39x csv.reader (control) 0.887 ms -> 0.887 ms 1.00x The maximum is cached on the dialect rather than recomputed per field so that the change never loses. Degenerate inputs (rows of empty fields, or a 4096-character lineterminator) measure 1.00-1.01x, and a long terminator with short fields improves from 0.11x to 2.94x against a per-field variant. --- ...-09-02-14-30-00.gh-issue-156955.Kv3Qa1.rst | 3 ++ Modules/_csv.c | 41 +++++++++++++++++-- 2 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-09-02-14-30-00.gh-issue-156955.Kv3Qa1.rst diff --git a/Misc/NEWS.d/next/Library/2026-09-02-14-30-00.gh-issue-156955.Kv3Qa1.rst b/Misc/NEWS.d/next/Library/2026-09-02-14-30-00.gh-issue-156955.Kv3Qa1.rst new file mode 100644 index 000000000000000..b0c058771451aa0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-02-14-30-00.gh-issue-156955.Kv3Qa1.rst @@ -0,0 +1,3 @@ +Speed up :func:`csv.writer` by up to 2.9x. Writing a field no longer calls +:c:func:`PyUnicode_FindChar` on the dialect's ``lineterminator`` once per +character. diff --git a/Modules/_csv.c b/Modules/_csv.c index c640f2d36a84647..e0e5c3050cefe57 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -116,6 +116,7 @@ typedef struct { Py_UCS4 delimiter; /* field separator */ Py_UCS4 quotechar; /* quote character */ Py_UCS4 escapechar; /* escape character */ + Py_UCS4 lineterm_maxchar; /* highest code point in lineterminator */ PyObject *lineterminator; /* string to write between records */ } DialectObj; @@ -332,6 +333,22 @@ _set_str(const char *name, PyObject **target, PyObject *src, const char *dflt) return 0; } +static Py_UCS4 +str_maxchar(PyObject *s) +{ + int kind = PyUnicode_KIND(s); + const void *data = PyUnicode_DATA(s); + Py_ssize_t len = PyUnicode_GET_LENGTH(s); + Py_UCS4 maxchar = 0; + for (Py_ssize_t i = 0; i < len; i++) { + Py_UCS4 c = PyUnicode_READ(kind, data, i); + if (c > maxchar) { + maxchar = c; + } + } + return maxchar; +} + static int dialect_check_quoting(int quoting) { @@ -533,6 +550,7 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) DIASET(_set_bool, "skipinitialspace", &self->skipinitialspace, skipinitialspace, false); DIASET(_set_bool, "strict", &self->strict, strict, false); #undef DIASET + self->lineterm_maxchar = str_maxchar(self->lineterminator); /* validate options */ if (dialect_check_quoting(self->quoting)) @@ -1165,6 +1183,21 @@ join_reset(WriterObj *self) #define MEM_INCR 32768 +static inline int +in_lineterminator(Py_UCS4 c, DialectObj *dialect) +{ + PyObject *lt = dialect->lineterminator; + int kind = PyUnicode_KIND(lt); + const void *data = PyUnicode_DATA(lt); + Py_ssize_t len = PyUnicode_GET_LENGTH(lt); + for (Py_ssize_t i = 0; i < len; i++) { + if (PyUnicode_READ(kind, data, i) == c) { + return 1; + } + } + return 0; +} + /* Calculate new record length or append field to record. Return new * record length. */ @@ -1176,6 +1209,10 @@ join_append_data(WriterObj *self, int field_kind, const void *field_data, DialectObj *dialect = self->dialect; Py_ssize_t i; Py_ssize_t rec_len; + /* A character above this cannot be in the line terminator, so the + scan below is skipped; the default "\r\n" rejects all ordinary + text that way. */ + Py_UCS4 term_maxchar = dialect->lineterm_maxchar; #define INCLEN \ do {\ @@ -1213,9 +1250,7 @@ join_append_data(WriterObj *self, int field_kind, const void *field_data, c == dialect->quotechar || c == '\n' || c == '\r' || - PyUnicode_FindChar( - dialect->lineterminator, c, 0, - PyUnicode_GET_LENGTH(dialect->lineterminator), 1) >= 0) { + (c <= term_maxchar && in_lineterminator(c, dialect))) { if (dialect->quoting == QUOTE_NONE) want_escape = 1; else {