fix replay fallback for missing content-encoding

Author: thaaddeus

tardis_dev/_http.py

@@ -118,7 +118,7 @@ async def _download(
             content_encoding = response.headers.get("Content-Encoding")
             if content_encoding == "zstd":
                 final_path = f"{dest_path}.zst"
-            elif content_encoding == "gzip":
+            elif content_encoding is None or content_encoding == "gzip":
                 final_path = f"{dest_path}.gz"
             else:
                 raise urllib.error.HTTPError(

tests/test_http.py

@@ -30,6 +30,22 @@ async def test_reliable_download_appends_zstd_extension_for_replay_cache(tmp_pat
     assert not destination.exists()
 
 
+@pytest.mark.asyncio
+async def test_reliable_download_falls_back_to_gzip_extension_when_content_encoding_is_missing(tmp_path: Path):
+    destination = tmp_path / "slice.json"
+    url = "https://example.com/data"
+
+    with aioresponses() as mocked:
+        mocked.get(url, body=b"")
+
+        async with await create_session("", 5) as session:
+            final_path = await reliable_download(session, url, str(destination), append_content_encoding_extension=True)
+
+    assert final_path.endswith(".gz")
+    assert Path(final_path).read_bytes() == b""
+    assert not destination.exists()
+
+
 @pytest.mark.asyncio
 async def test_reliable_download_uses_node_backoff_for_500(tmp_path: Path, monkeypatch):
     destination = tmp_path / "slice.json.gz"

tests/test_replay.py

@@ -20,12 +20,19 @@
 LIVE_REPLAY_EXCHANGE = "bitmex"
 LIVE_REPLAY_FROM = "2019-05-01T00:00:00.000Z"
 LIVE_REPLAY_TO = "2019-05-01T00:01:00.000Z"
+LIVE_EMPTY_REPLAY_EXCHANGE = "binance"
+LIVE_EMPTY_REPLAY_FROM = "2019-06-01T00:00:00.000Z"
+LIVE_EMPTY_REPLAY_TO = "2019-06-01T00:01:00.000Z"
 
 
 def _live_replay_filters():
     return [Channel("trade", ["BTCUSD"])]
 
 
+def _live_empty_replay_filters():
+    return [Channel("trade", ["batpax"])]
+
+
 class _FakeSession:
     async def __aenter__(self):
         return object()
@@ -90,6 +97,23 @@ async def test_replay_auto_cleanup_removes_live_processed_slices(tmp_path: Path)
     assert not day_dir.exists()
 
 
+@pytest.mark.live
+@pytest.mark.asyncio
+async def test_replay_live_empty_slice_yields_no_messages(tmp_path: Path):
+    cache_dir = tmp_path / "cache"
+    results = []
+    async for item in replay(
+        exchange=LIVE_EMPTY_REPLAY_EXCHANGE,
+        from_date=LIVE_EMPTY_REPLAY_FROM,
+        to_date=LIVE_EMPTY_REPLAY_TO,
+        filters=_live_empty_replay_filters(),
+        cache_dir=str(cache_dir),
+    ):
+        results.append(item)
+
+    assert results == []
+
+
 def test_replay_rejects_invalid_date_order():
     async def collect():
         async for _ in replay(exchange="bitmex", from_date="2019-06-02", to_date="2019-06-01"):