From 3371b35d1a15856a73bd8c4f9f7d9fb837e11503 Mon Sep 17 00:00:00 2001 From: Numan Karaaslan Date: Wed, 15 Jul 2026 12:53:35 +0300 Subject: [PATCH 1/3] feat: Removed wordlift referrer from http requests --- AGENTS.md | 2 +- README.md | 2 +- wordlift_sdk/render/render_options.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 74a8f4c..814703a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -137,7 +137,7 @@ implicit source auto-priority are intentionally removed. - Sitemap ingestion source discovery (`SitemapSourceAdapter`) sets browser-like request headers for `advertools.sitemaps.sitemap_to_df` - (`User-Agent`, `Accept`, `Accept-Language`, `Referer`, + (`User-Agent`, `Accept`, `Accept-Language`, `Upgrade-Insecure-Requests`, `Sec-CH-*`) so sitemap fetch identity is closer to Playwright/browser ingestion defaults. - `wordlift_sdk.kg_build` callback patch preparation annotates first-level diff --git a/README.md b/README.md index 3e9a9ab..1c20d89 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Current release: see `CHANGELOG.md`. ## Features - URL sources: XML sitemaps, Google Sheets (`url` column), or Python lists, with global optional `URL_REGEX` filtering (also enforced in graph-sync source selection before `new_or_changed` GraphQL lookup). -- Sitemap discovery requests use a browser-like header bundle aligned with Playwright defaults (including `User-Agent`, `Accept`, `Accept-Language`, `Referer`, and `Sec-CH-*` headers). +- Sitemap discovery requests use a browser-like header bundle aligned with Playwright defaults (including `User-Agent`, `Accept`, `Accept-Language`, and `Sec-CH-*` headers). - Change detection: skips URLs that are already imported unless `OVERWRITE` is enabled; re-imports when `lastmod` is newer. - Web page imports: sends URLs to WordLift with embedding requests, output types, retry logic, and pluggable callbacks. - Python 3.14 compatibility: retry filters use `pydantic_core.ValidationError` via the public API. diff --git a/wordlift_sdk/render/render_options.py b/wordlift_sdk/render/render_options.py index 9383bf8..44c9ac3 100644 --- a/wordlift_sdk/render/render_options.py +++ b/wordlift_sdk/render/render_options.py @@ -18,7 +18,6 @@ DEFAULT_BROWSER_REQUEST_HEADERS = { "Accept": DEFAULT_ACCEPT_HEADER, "Accept-Language": DEFAULT_ACCEPT_LANGUAGE_HEADER, - "Referer": "https://wordlift.io", "Upgrade-Insecure-Requests": "1", "Sec-CH-UA": '"Not A(Brand";v="99", "Chromium";v="120", "Google Chrome";v="120"', "Sec-CH-UA-Mobile": "?0", From ad0e348cf07fee5a5d2f85a8f05e6aad0a015fdf Mon Sep 17 00:00:00 2001 From: Numan Karaaslan Date: Wed, 15 Jul 2026 12:54:00 +0300 Subject: [PATCH 2/3] test: Added tests for referrer removal --- tests/ingestion/test_loaders.py | 38 +++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/ingestion/test_loaders.py b/tests/ingestion/test_loaders.py index 3cf465a..34fe3d0 100644 --- a/tests/ingestion/test_loaders.py +++ b/tests/ingestion/test_loaders.py @@ -130,6 +130,44 @@ def _urlopen(request, timeout): } +def test_simple_loader_sends_no_referer(monkeypatch: pytest.MonkeyPatch) -> None: + """Regression: a real fetch must not carry a wordlift.io Referer. + + The equality check above only proves the loader forwards the builder's + headers; it would still pass if the referer were re-added to the builder. + This asserts absence on the actual outgoing request. + """ + loader = SimpleLoaderAdapter() + observed: dict[str, str] = {} + + class _Resp(BytesIO): + status = 200 + + def __init__(self) -> None: + super().__init__(b"ok") + + def geturl(self) -> str: + return "https://example.com/final" + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def _urlopen(request, timeout): + observed.update( + {key.lower(): value for key, value in dict(request.header_items()).items()} + ) + return _Resp() + + monkeypatch.setattr("urllib.request.urlopen", _urlopen) + loader.load(SourceItem(id="1", url="https://example.com"), _config()) + + assert "referer" not in observed + assert "https://wordlift.io" not in observed.values() + + def test_simple_loader_adapter_wraps_url_errors( monkeypatch: pytest.MonkeyPatch, ) -> None: From 913bed2be464f60ad8858c4d9fe0fefe40c1646f Mon Sep 17 00:00:00 2001 From: Numan Karaaslan Date: Wed, 15 Jul 2026 12:54:58 +0300 Subject: [PATCH 3/3] chore: Bump version to 8.3.8 --- CHANGELOG.md | 8 ++++++++ pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aab0aab..1f6823a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 8.3.8 - 2026-07-15 + +### Changed + +- Removed the `Referer: https://wordlift.io` header from browser-like fetch + requests (`simple` / `playwright` loaders and sitemap discovery), so client + sites no longer see `wordlift.io` referral traffic in their analytics. + ## 8.3.7 - 2026-07-13 ### Fixed diff --git a/pyproject.toml b/pyproject.toml index eebeb03..1fada22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wordlift-sdk" -version = "8.3.7" +version = "8.3.8" description = "Python toolkit for orchestrating WordLift imports and structured data workflows." authors = ["David Riccitelli "] readme = "README.md"