From 177d3c8cdfc029c0c2f7cd3d1e87ab61afd48d7a Mon Sep 17 00:00:00 2001 From: Muse Sweep Date: Wed, 21 Oct 2026 19:09:13 -0500 Subject: [PATCH] fix(lab-ops-hub): refuse DEL/C1/NBSP and WHATWG-forbidden hosts in container validator (#510) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #507 check stopped at 0x20: DEL (0x7F), the C1 controls (0x80-0x9F), NBSP (0xA0) and the WHATWG forbidden host codepoints < > ^ | still fell through validate_crawl_url as 'a public DNS name', while the Worker's validateCrawlUrl refuses every one. Extend the up-front check to ord <= 0x20 or 0x7F <= ord <= 0xA0 or one of <>^|. Non-ASCII above NBSP (valid IDN hosts) is untouched. New regression test covers each refused class plus IDN/underscore controls; it fails with only container/app.py reverted. Differential probe over the issue's battery (printable ASCII + 0x7F-0xA0 + é + combining char) now shows full TS/Python parity. --- pkgs/lab-ops-hub/container/app.py | 16 ++++++++----- .../test/test_container_validate.py | 23 +++++++++++++++++++ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/pkgs/lab-ops-hub/container/app.py b/pkgs/lab-ops-hub/container/app.py index 810f557ce..9ebd61e3a 100644 --- a/pkgs/lab-ops-hub/container/app.py +++ b/pkgs/lab-ops-hub/container/app.py @@ -65,12 +65,16 @@ def validate_crawl_url(raw): # the zone id). Refuse every % host, fail-closed (#505). if "%" in host: raise ValueError(f"refusing to crawl percent-encoded host {host}") - # A space or control character in the host is not a DNS name: WHATWG - # rejects it outright, urlparse hands it through verbatim, and - # http.client only raises for it deep inside the fetcher — the same - # lazy-failure shape #502 fixed for ports. Refuse up front (#507). - if any(ord(c) <= 0x20 for c in host): - raise ValueError(f"refusing to crawl host with whitespace {host!r}") + # A space, control or WHATWG-forbidden character in the host is not + # a DNS name: WHATWG rejects it outright, urlparse hands it through + # verbatim, and http.client only raises for it deep inside the + # fetcher — the same lazy-failure shape #502 fixed for ports. Refuse + # up front: C0 controls and space (#507), plus DEL, the C1 controls + # and NBSP (0x7F-0xA0) and the forbidden host codepoints < > ^ | + # (#510). Non-ASCII above NBSP (a valid IDN host, e.g. é) is + # untouched. + if any(ord(c) <= 0x20 or 0x7F <= ord(c) <= 0xA0 or c in "<>^|" for c in host): + raise ValueError(f"refusing to crawl host with a forbidden character {host!r}") if host == "localhost" or host.endswith(_BLOCKED_SUFFIXES): raise ValueError(f"refusing to crawl local host {host}") try: diff --git a/pkgs/lab-ops-hub/test/test_container_validate.py b/pkgs/lab-ops-hub/test/test_container_validate.py index 2e2cd463a..b7ba8c7b0 100644 --- a/pkgs/lab-ops-hub/test/test_container_validate.py +++ b/pkgs/lab-ops-hub/test/test_container_validate.py @@ -248,6 +248,29 @@ def test_refuses_space_hosts_and_ipvfuture_literals(self): ): self.assertRefused(url) + def test_refuses_del_c1_nbsp_and_forbidden_host_characters(self): + # The #507 check stopped at 0x20, one codepoint short of + # http.client's own set and short of the rest of the WHATWG + # forbidden-host set: DEL, the C1 controls, NBSP and < > ^ | + # still fell through as "a public DNS name", while the + # Worker's WHATWG validator refuses every one (#510). + for url in ( + "http://exam\x7fple.com/", # DEL + "http://exam\x80ple.com/", # C1 control, low end + "http://exam\x85ple.com/", # C1 control (NEL) + "http://exam\x9fple.com/", # C1 control, high end + "http://exam\xa0ple.com/", # NBSP + "http://example.com/", + "http://exam^ple.com/", + "http://exam|ple.com/", + ): + self.assertRefused(url) + # Controls: characters above NBSP and ordinary DNS punctuation + # are not forbidden — valid IDN hosts stay crawlable. + self.assertAccepted("http://exam_ple.com/") + self.assertAccepted("http://examéple.com/") + def test_accepts_public_targets(self): for url in ( "https://example.com/page",