From ada363eff01b578ced4cc208e59416df932161de Mon Sep 17 00:00:00 2001 From: Zac Farrell Date: Fri, 18 Sep 2026 12:37:44 -0700 Subject: [PATCH 1/3] feat(transport): set compression and User-Agent request defaults The generated client left two server-observable defaults to urllib3. urllib3 puts `Accept-Encoding: identity` on every connection, which is not "no preference" but an explicit request not to compress, so every JSON response came back uncompressed. RESTClientObject.request now advertises urllib3.util.request.ACCEPT_ENCODING -- exactly the codecs the installed urllib3 can transparently decode, so a server can never negotiate an encoding that arrives as undecodable bytes. It is a setdefault, so an operation whose payload is already compressed can still pass `identity`. Requests also identified themselves as OpenAPI-Generator/1.0.0/python, which attributes traffic to neither the SDK nor a release. The generator's httpUserAgent property only bakes a literal at generation time, and regens follow spec changes rather than releases, so a baked version would go stale between them; the string is resolved at import time from installed package metadata instead and lives in the generator-ignored hotdata/_useragent.py. rest.py and api_client.py are generator output, so the edits are re-applied by scripts/patch_request_defaults.py, wired into regenerate.yml alongside the existing patch steps. --- .github/workflows/regenerate.yml | 3 + .openapi-generator-ignore | 5 +- CHANGELOG.md | 10 ++ hotdata/_useragent.py | 52 +++++++++ hotdata/api_client.py | 3 +- hotdata/rest.py | 14 +++ scripts/patch_request_defaults.py | 97 +++++++++++++++++ tests/test_request_defaults.py | 169 ++++++++++++++++++++++++++++++ 8 files changed, 351 insertions(+), 2 deletions(-) create mode 100644 hotdata/_useragent.py create mode 100755 scripts/patch_request_defaults.py create mode 100644 tests/test_request_defaults.py diff --git a/.github/workflows/regenerate.yml b/.github/workflows/regenerate.yml index bfea8a0..e6b81f4 100644 --- a/.github/workflows/regenerate.yml +++ b/.github/workflows/regenerate.yml @@ -198,6 +198,9 @@ jobs: - name: Patch default client exports (enhanced query/results) run: python3 scripts/patch_query_exports.py + - name: Patch transport request defaults (Accept-Encoding, User-Agent) + run: python3 scripts/patch_request_defaults.py + # The API-token -> JWT key exchange is deprecated: the configured API token # is the bearer credential and goes on the wire verbatim. This check is the # inverse of the old "did the exchange survive?" guard — it fails if the diff --git a/.openapi-generator-ignore b/.openapi-generator-ignore index eb2a6c6..fe949af 100644 --- a/.openapi-generator-ignore +++ b/.openapi-generator-ignore @@ -9,11 +9,14 @@ setup.py # truth for "hand-maintained, don't touch": arrow.py (Arrow IPC result fetch), # query.py (429 retry + truncation auto-follow, #688), _retry.py (pre-response # connection-reset retry on all methods, #118), uploads.py (transparent -# presigned direct-to-storage upload flow). +# presigned direct-to-storage upload flow), _useragent.py (runtime-resolved +# User-Agent; the generator's httpUserAgent property can only bake a literal at +# generation time, which goes stale between releases). hotdata/arrow.py hotdata/query.py hotdata/_retry.py hotdata/uploads.py +hotdata/_useragent.py # Hand-written test for the patched ApiClient.close()/context-manager behavior # (re-applied by scripts/patch_api_client_close.py). It lives in the generated diff --git a/CHANGELOG.md b/CHANGELOG.md index c2a1fc4..6871f4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 compatibility. Generated docstrings pick up the new wording on the next client regeneration from the updated OpenAPI spec. - feat(query): add dialect parameter to query request +- perf(transport): request compressed responses. urllib3 defaults every + connection to `Accept-Encoding: identity`, which asks the server *not* to + compress, so every JSON response came back uncompressed. The client now + advertises `urllib3.util.request.ACCEPT_ENCODING` — exactly the codecs the + installed urllib3 can transparently decode. Response bodies are unchanged; + a caller or operation can still pass an explicit `Accept-Encoding`. +- feat(transport): send an SDK `User-Agent`. Requests identified themselves as + `OpenAPI-Generator/1.0.0/python`; they now send + `hotdata-python/ (Python/; urllib3/)`. Setting + `ApiClient.user_agent` still overrides it. ## [0.10.0] - 2026-08-18 diff --git a/hotdata/_useragent.py b/hotdata/_useragent.py new file mode 100644 index 0000000..509e6c7 --- /dev/null +++ b/hotdata/_useragent.py @@ -0,0 +1,52 @@ +"""The SDK's default ``User-Agent``. + +The generator's default is ``OpenAPI-Generator/1.0.0/python``, which identifies +neither the SDK nor its version — server-side telemetry cannot tell a hotdata +client from any other generated client, let alone one release from another. + +openapi-generator *does* expose an ``httpUserAgent`` property for this, but it +bakes a literal string in at **generation** time. Regeneration is driven by +OpenAPI spec changes, not by releases, so a baked version string reports +whatever the version happened to be at the last regen — it would go stale +silently and misattribute traffic, which is worse than the generic default. The +version is therefore resolved at import time from installed package metadata, +the same source :mod:`hotdata.__init__` uses for ``__version__``. + +This lives outside the generated modules (see ``.openapi-generator-ignore``) so +a regeneration cannot overwrite it; ``scripts/patch_request_defaults.py`` only +has to re-point ``ApiClient`` at the constant. +""" + +from __future__ import annotations + +import importlib.metadata +import platform + +import urllib3 + + +def _default_user_agent() -> str: + """``hotdata-python/ (Python/; urllib3/)``. + + The runtime versions ride along because the transport stack is where + client-side failures usually originate: knowing which urllib3 a bug report + came from is the difference between reproducing a problem and guessing. + """ + try: + sdk_version = importlib.metadata.version("hotdata") + except importlib.metadata.PackageNotFoundError: + # Running from a source checkout without an install. + sdk_version = "0.0.0+unknown" + return ( + f"hotdata-python/{sdk_version} " + f"(Python/{platform.python_version()}; urllib3/{urllib3.__version__})" + ) + + +#: Sent as ``User-Agent`` on every request. ``ApiClient.user_agent`` still +#: overrides it per client, which is the supported way for an application to +#: identify itself. +USER_AGENT = _default_user_agent() + + +__all__ = ["USER_AGENT", "_default_user_agent"] diff --git a/hotdata/api_client.py b/hotdata/api_client.py index 1580352..b9f119f 100644 --- a/hotdata/api_client.py +++ b/hotdata/api_client.py @@ -29,6 +29,7 @@ from hotdata.configuration import Configuration from hotdata.api_response import ApiResponse, T as ApiResponseT +from hotdata._useragent import USER_AGENT import hotdata.models from hotdata import rest from hotdata.exceptions import ( @@ -91,7 +92,7 @@ def __init__( self.default_headers[header_name] = header_value self.cookie = cookie # Set default User-Agent. - self.user_agent = 'OpenAPI-Generator/1.0.0/python' + self.user_agent = USER_AGENT self.client_side_validation = configuration.client_side_validation def __enter__(self): diff --git a/hotdata/rest.py b/hotdata/rest.py index 33f704e..d23fa2d 100644 --- a/hotdata/rest.py +++ b/hotdata/rest.py @@ -19,6 +19,7 @@ import ssl import urllib3 +from urllib3.util.request import ACCEPT_ENCODING from hotdata.exceptions import ApiException, ApiValueError @@ -164,6 +165,19 @@ def request( post_params = post_params or {} headers = headers or {} + # Ask for compressed responses. urllib3 defaults every connection to + # `Accept-Encoding: identity`, which is not "no preference" but an + # explicit request *not* to compress, and a spec-compliant server + # honors it. ACCEPT_ENCODING is built from the codecs the installed + # urllib3 can actually decode (gzip/deflate, plus br/zstd when their + # backends are present), so the server can never negotiate an encoding + # that reaches us as undecodable bytes. urllib3 decodes the body + # transparently, so callers are unaffected. + # + # setdefault, not assignment: an operation whose payload is already + # compressed end-to-end can pass `identity` and stay in control. + headers.setdefault('Accept-Encoding', ACCEPT_ENCODING) + timeout = None if _request_timeout: if isinstance(_request_timeout, (int, float)): diff --git a/scripts/patch_request_defaults.py b/scripts/patch_request_defaults.py new file mode 100755 index 0000000..e967039 --- /dev/null +++ b/scripts/patch_request_defaults.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Re-apply the SDK's transport request defaults after OpenAPI regeneration. + +Two defaults the generator leaves on urllib3's (server-observable) behavior: + +* ``Accept-Encoding``: urllib3 puts ``identity`` on every connection, which + asks the server *not* to compress. We advertise + ``urllib3.util.request.ACCEPT_ENCODING`` instead — exactly the codecs the + installed urllib3 can transparently decode. +* ``User-Agent``: ``OpenAPI-Generator/1.0.0/python`` identifies neither the SDK + nor its version, so server-side telemetry cannot attribute traffic. +""" + +from __future__ import annotations + +import pathlib +import sys + +ROOT = pathlib.Path(__file__).resolve().parents[1] + + +def patch_accept_encoding() -> None: + """Advertise compression on every request made through RESTClientObject.""" + path = ROOT / "hotdata" / "rest.py" + src = path.read_text() + + if "ACCEPT_ENCODING" in src: + return + + import_needle = "import urllib3\n" + import_replacement = "import urllib3\nfrom urllib3.util.request import ACCEPT_ENCODING\n" + if import_needle not in src: + sys.exit(f"Failed to patch {path}: urllib3 import anchor not found") + src = src.replace(import_needle, import_replacement, 1) + + needle = " post_params = post_params or {}\n headers = headers or {}\n" + replacement = ( + " post_params = post_params or {}\n" + " headers = headers or {}\n\n" + " # Ask for compressed responses. urllib3 defaults every connection to\n" + " # `Accept-Encoding: identity`, which is not \"no preference\" but an\n" + " # explicit request *not* to compress, and a spec-compliant server\n" + " # honors it. ACCEPT_ENCODING is built from the codecs the installed\n" + " # urllib3 can actually decode (gzip/deflate, plus br/zstd when their\n" + " # backends are present), so the server can never negotiate an encoding\n" + " # that reaches us as undecodable bytes. urllib3 decodes the body\n" + " # transparently, so callers are unaffected.\n" + " #\n" + " # setdefault, not assignment: an operation whose payload is already\n" + " # compressed end-to-end can pass `identity` and stay in control.\n" + " headers.setdefault('Accept-Encoding', ACCEPT_ENCODING)\n" + ) + if needle not in src: + sys.exit(f"Failed to patch {path}: request() header anchor not found") + src = src.replace(needle, replacement, 1) + + path.write_text(src) + + +def patch_user_agent() -> None: + """Point ApiClient at the hand-maintained User-Agent constant. + + The string itself lives in ``hotdata/_useragent.py`` (generator-ignored), so + this patch is just an import plus the assignment — two anchors instead of + carrying the logic inside generated output. + """ + path = ROOT / "hotdata" / "api_client.py" + src = path.read_text() + + if "_useragent" in src: + return + + import_needle = "from hotdata.api_response import ApiResponse, T as ApiResponseT\n" + import_replacement = ( + "from hotdata.api_response import ApiResponse, T as ApiResponseT\n" + "from hotdata._useragent import USER_AGENT\n" + ) + if import_needle not in src: + sys.exit(f"Failed to patch {path}: api_response import anchor not found") + src = src.replace(import_needle, import_replacement, 1) + + ua_needle = " self.user_agent = 'OpenAPI-Generator/1.0.0/python'\n" + ua_replacement = " self.user_agent = USER_AGENT\n" + if ua_needle not in src: + sys.exit(f"Failed to patch {path}: default User-Agent anchor not found") + src = src.replace(ua_needle, ua_replacement, 1) + + path.write_text(src) + + +def main() -> None: + patch_accept_encoding() + patch_user_agent() + + +if __name__ == "__main__": + main() diff --git a/tests/test_request_defaults.py b/tests/test_request_defaults.py new file mode 100644 index 0000000..d6e3ae8 --- /dev/null +++ b/tests/test_request_defaults.py @@ -0,0 +1,169 @@ +"""Transport-level request defaults the SDK puts on every call. + +Two things the generated client left on urllib3's defaults, both observable by +the server and neither of them what an SDK wants: + +* **Compression.** urllib3 sets ``Accept-Encoding: identity`` on every + connection it opens. That is not "no preference" — it is an explicit request + *not* to compress, and a spec-compliant server honors it. Every JSON response + (query results, table listings, job payloads) therefore came back + uncompressed. The fix advertises ``urllib3.util.request.ACCEPT_ENCODING``, + which is exactly the set the *installed* urllib3 can transparently decode, so + the SDK can never be handed a body it cannot read. + +* **User-Agent.** ``OpenAPI-Generator/1.0.0/python`` identifies neither the SDK + nor its version, so server-side telemetry cannot tell one client release from + another — or a hotdata client from any other generated client. + +These tests drive the real ``ApiClient`` against a real socket and assert on +what the *server* saw, because the header is only interesting on the wire: a +unit assertion against ``default_headers`` would have passed both before and +after the fix. +""" + +from __future__ import annotations + +import gzip +import json +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer +from typing import Any + +import pytest +from urllib3.util.request import ACCEPT_ENCODING + +from hotdata.api_client import ApiClient +from hotdata.configuration import Configuration + + +class _RecordingHandler(BaseHTTPRequestHandler): + """Records the request headers, then replies with a gzipped JSON body. + + The reply is compressed unconditionally — the point is to prove the client + both *asks* for compression and transparently *decodes* the result, so the + body is gzipped regardless of what the client advertised. + """ + + protocol_version = "HTTP/1.1" + payload = {"rows": ["value"] * 64} + + def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + self.server.seen_headers = { # type: ignore[attr-defined] + key.lower(): value for key, value in self.headers.items() + } + body = gzip.compress(json.dumps(self.payload).encode()) + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Encoding", "gzip") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args: Any) -> None: + """Silence the stderr request log.""" + + +class _QuietServer(HTTPServer): + """An HTTPServer that does not log the teardown reset. + + The client holds a keep-alive connection open and drops it when the + ``ApiClient`` context manager exits, which the handler thread sees as a + reset. That is the expected end of the test, not a failure, so it should + not print a traceback. + """ + + def handle_error(self, request: Any, client_address: Any) -> None: + """Swallow the expected connection reset at teardown.""" + + +@pytest.fixture +def server(): + """A local HTTP server that records what the client sent.""" + httpd = _QuietServer(("127.0.0.1", 0), _RecordingHandler) + httpd.seen_headers = {} # type: ignore[attr-defined] + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + try: + yield httpd + finally: + httpd.shutdown() + httpd.server_close() + thread.join(timeout=5) + + +@pytest.fixture +def client(server): + """A real ApiClient pointed at the recording server.""" + host = f"http://127.0.0.1:{server.server_address[1]}" + with ApiClient(Configuration(host=host)) as api_client: + yield api_client + + +def _get(client: ApiClient, server: HTTPServer, headers: dict[str, str] | None = None) -> Any: + """Issue a GET through the full client stack and return the response.""" + host = f"http://127.0.0.1:{server.server_address[1]}" + header_params = dict(client.default_headers) + header_params.update(headers or {}) + return client.call_api("GET", f"{host}/v1/results", header_params=header_params) + + +def test_requests_compressed_responses(client, server): + """The SDK asks for compression instead of urllib3's `identity` opt-out.""" + _get(client, server) + + accept_encoding = server.seen_headers["accept-encoding"] + assert accept_encoding != "identity" + assert "gzip" in accept_encoding + + +def test_advertises_only_encodings_urllib3_can_decode(client, server): + """Never advertise an encoding the installed urllib3 cannot decode. + + ``ACCEPT_ENCODING`` is built from the codecs actually available at import + time (brotli and zstd are optional), so pinning to it — rather than to a + hardcoded string — is what guarantees a server can never pick an encoding + that would come back as undecodable bytes. + """ + _get(client, server) + + assert server.seen_headers["accept-encoding"] == ACCEPT_ENCODING + + +def test_compressed_response_is_transparently_decoded(client, server): + """Asking for compression must not change what callers receive.""" + response = _get(client, server) + + assert json.loads(response.read()) == _RecordingHandler.payload + + +def test_caller_can_override_accept_encoding(client, server): + """An explicit per-request encoding wins over the default. + + A response that is already compressed end-to-end (an Arrow IPC stream with + LZ4/ZSTD record batches, say) gains nothing from a second pass, so the + default must be a floor, not a ceiling. + """ + _get(client, server, headers={"Accept-Encoding": "identity"}) + + assert server.seen_headers["accept-encoding"] == "identity" + + +def test_user_agent_identifies_the_sdk_and_version(client, server): + """Server-side telemetry must be able to attribute traffic to a release.""" + from hotdata.api_client import USER_AGENT + + _get(client, server) + + user_agent = server.seen_headers["user-agent"] + assert user_agent == USER_AGENT + assert "OpenAPI-Generator" not in user_agent + assert user_agent.startswith("hotdata-python/") + + +def test_user_agent_remains_caller_settable(client, server): + """The generator's `user_agent` setter stays the supported override.""" + client.user_agent = "my-app/2.0" + + _get(client, server) + + assert server.seen_headers["user-agent"] == "my-app/2.0" From af4c7c1bbc3416bdbc30211bc3d521a8179593c2 Mon Sep 17 00:00:00 2001 From: Zac Farrell Date: Fri, 18 Sep 2026 12:47:49 -0700 Subject: [PATCH 2/3] fix(transport): match an Accept-Encoding override case-insensitively Header names are case-insensitive, so a caller passing `accept-encoding` left both keys in the dict. urllib3 emits one header line per key, so the server received the opt-out *and* the compressed set and could still compress -- silently losing the documented per-request escape hatch. Compare lowercased, the way urllib3 does before adding its own Accept-Encoding. --- CHANGELOG.md | 4 +++- hotdata/rest.py | 12 +++++++++--- scripts/patch_request_defaults.py | 12 +++++++++--- tests/test_request_defaults.py | 22 ++++++++++++++++++++++ 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6871f4f..91d061f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 compress, so every JSON response came back uncompressed. The client now advertises `urllib3.util.request.ACCEPT_ENCODING` — exactly the codecs the installed urllib3 can transparently decode. Response bodies are unchanged; - a caller or operation can still pass an explicit `Accept-Encoding`. + a caller or operation can still pass an explicit `Accept-Encoding` (matched + case-insensitively). Arrow IPC fetches opt out and stay uncompressed, since + their record batches are frequently compressed by the writer already. - feat(transport): send an SDK `User-Agent`. Requests identified themselves as `OpenAPI-Generator/1.0.0/python`; they now send `hotdata-python/ (Python/; urllib3/)`. Setting diff --git a/hotdata/rest.py b/hotdata/rest.py index d23fa2d..4531376 100644 --- a/hotdata/rest.py +++ b/hotdata/rest.py @@ -174,9 +174,15 @@ def request( # that reaches us as undecodable bytes. urllib3 decodes the body # transparently, so callers are unaffected. # - # setdefault, not assignment: an operation whose payload is already - # compressed end-to-end can pass `identity` and stay in control. - headers.setdefault('Accept-Encoding', ACCEPT_ENCODING) + # A default, not an override: an operation whose payload is already + # compressed end-to-end passes `identity` and stays in control. The + # check is case-insensitive because header names are -- a caller + # passing `accept-encoding` would otherwise leave both keys in the + # dict and urllib3 would emit two header lines, so the server would + # see the opt-out *and* the compressed set. urllib3 lowercases names + # the same way before adding its own Accept-Encoding. + if not any(key.lower() == 'accept-encoding' for key in headers): + headers['Accept-Encoding'] = ACCEPT_ENCODING timeout = None if _request_timeout: diff --git a/scripts/patch_request_defaults.py b/scripts/patch_request_defaults.py index e967039..1c3626e 100755 --- a/scripts/patch_request_defaults.py +++ b/scripts/patch_request_defaults.py @@ -46,9 +46,15 @@ def patch_accept_encoding() -> None: " # that reaches us as undecodable bytes. urllib3 decodes the body\n" " # transparently, so callers are unaffected.\n" " #\n" - " # setdefault, not assignment: an operation whose payload is already\n" - " # compressed end-to-end can pass `identity` and stay in control.\n" - " headers.setdefault('Accept-Encoding', ACCEPT_ENCODING)\n" + " # A default, not an override: an operation whose payload is already\n" + " # compressed end-to-end passes `identity` and stays in control. The\n" + " # check is case-insensitive because header names are -- a caller\n" + " # passing `accept-encoding` would otherwise leave both keys in the\n" + " # dict and urllib3 would emit two header lines, so the server would\n" + " # see the opt-out *and* the compressed set. urllib3 lowercases names\n" + " # the same way before adding its own Accept-Encoding.\n" + " if not any(key.lower() == 'accept-encoding' for key in headers):\n" + " headers['Accept-Encoding'] = ACCEPT_ENCODING\n" ) if needle not in src: sys.exit(f"Failed to patch {path}: request() header anchor not found") diff --git a/tests/test_request_defaults.py b/tests/test_request_defaults.py index d6e3ae8..0cd7325 100644 --- a/tests/test_request_defaults.py +++ b/tests/test_request_defaults.py @@ -51,6 +51,11 @@ def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API self.server.seen_headers = { # type: ignore[attr-defined] key.lower(): value for key, value in self.headers.items() } + # get_all, not get: a case-sensitive default check produces two + # Accept-Encoding header *lines*, which a dict of headers would hide. + self.server.seen_encodings = ( # type: ignore[attr-defined] + self.headers.get_all("Accept-Encoding") or [] + ) body = gzip.compress(json.dumps(self.payload).encode()) self.send_response(200) self.send_header("Content-Type", "application/json") @@ -81,6 +86,7 @@ def server(): """A local HTTP server that records what the client sent.""" httpd = _QuietServer(("127.0.0.1", 0), _RecordingHandler) httpd.seen_headers = {} # type: ignore[attr-defined] + httpd.seen_encodings = [] # type: ignore[attr-defined] thread = threading.Thread(target=httpd.serve_forever, daemon=True) thread.start() try: @@ -167,3 +173,19 @@ def test_user_agent_remains_caller_settable(client, server): _get(client, server) assert server.seen_headers["user-agent"] == "my-app/2.0" + + +def test_lowercase_override_is_not_duplicated(client, server): + """A differently-cased opt-out must suppress the default, not duplicate it. + + HTTP header names are case-insensitive and nothing stops a caller from + passing ``accept-encoding``. A case-*sensitive* default check leaves both + keys in the dict, urllib3 emits one header line per key, and the server + receives ``identity`` *and* the compressed set — so the documented + per-request opt-out is silently lost. urllib3 itself lowercases header + names before deciding whether to add its own ``Accept-Encoding``; this + matches that. + """ + _get(client, server, headers={"accept-encoding": "identity"}) + + assert server.seen_encodings == ["identity"] From c274c40ab98a57541df5feeaf6fae8b0151e19b3 Mon Sep 17 00:00:00 2001 From: Zac Farrell Date: Fri, 18 Sep 2026 12:47:55 -0700 Subject: [PATCH 3/3] perf(arrow): keep Arrow IPC responses uncompressed _call_arrow set only Accept, so the new client-wide compression default applied to Arrow fetches too. IPC record batches are frequently LZ4- or ZSTD-compressed by the writer already, making a gzip pass over the stream CPU on both ends for little size gain. Opt the path out with `Accept-Encoding: identity`, restoring the transfer behavior Arrow had before the default was introduced. --- hotdata/arrow.py | 11 ++++++++++- tests/test_arrow.py | 4 ++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/hotdata/arrow.py b/hotdata/arrow.py index 6e1d1c5..f2b39f6 100644 --- a/hotdata/arrow.py +++ b/hotdata/arrow.py @@ -174,7 +174,16 @@ def _call_arrow( # Override only what we need: the Accept header and the format query. # `GET /v1/results/{id}` is database-scoped, so the required # X-Database-Id header flows through the generated serializer too. - headers: Dict[str, Any] = {"Accept": ARROW_STREAM_MEDIA_TYPE} + # `Accept-Encoding: identity` opts this path out of the client-wide + # response compression default (hotdata/rest.py). Arrow IPC record + # batches are frequently LZ4/ZSTD-compressed by the writer already, so + # a gzip pass over the stream burns CPU on both ends for little size + # gain. Drop it if the endpoint is measured to serve uncompressed + # batches, where columnar data does compress well. + headers: Dict[str, Any] = { + "Accept": ARROW_STREAM_MEDIA_TYPE, + "Accept-Encoding": "identity", + } params = self._get_result_serialize( id=id, x_database_id=x_database_id, diff --git a/tests/test_arrow.py b/tests/test_arrow.py index ee5de0d..da035b2 100644 --- a/tests/test_arrow.py +++ b/tests/test_arrow.py @@ -163,6 +163,10 @@ def test_get_result_arrow_returns_table(monkeypatch: pytest.MonkeyPatch) -> None assert call["headers"]["Accept"] == ARROW_STREAM_MEDIA_TYPE # Results are database-scoped: the required X-Database-Id header is sent. assert call["headers"]["X-Database-Id"] == "db_x" + # Arrow opts out of the client-wide response compression default: IPC + # record batches are frequently LZ4/ZSTD-compressed already, so a gzip + # pass over the stream costs CPU on both ends for little size gain. + assert call["headers"]["Accept-Encoding"] == "identity" def test_get_result_arrow_forwards_offset_and_limit(