diff --git a/pkgs/lab-ops-hub/container/app.py b/pkgs/lab-ops-hub/container/app.py index 810f557ce..9ebd61e3a 100644 --- a/pkgs/lab-ops-hub/container/app.py +++ b/pkgs/lab-ops-hub/container/app.py @@ -65,12 +65,16 @@ def validate_crawl_url(raw): # the zone id). Refuse every % host, fail-closed (#505). if "%" in host: raise ValueError(f"refusing to crawl percent-encoded host {host}") - # A space or control character in the host is not a DNS name: WHATWG - # rejects it outright, urlparse hands it through verbatim, and - # http.client only raises for it deep inside the fetcher — the same - # lazy-failure shape #502 fixed for ports. Refuse up front (#507). - if any(ord(c) <= 0x20 for c in host): - raise ValueError(f"refusing to crawl host with whitespace {host!r}") + # A space, control or WHATWG-forbidden character in the host is not + # a DNS name: WHATWG rejects it outright, urlparse hands it through + # verbatim, and http.client only raises for it deep inside the + # fetcher — the same lazy-failure shape #502 fixed for ports. Refuse + # up front: C0 controls and space (#507), plus DEL, the C1 controls + # and NBSP (0x7F-0xA0) and the forbidden host codepoints < > ^ | + # (#510). Non-ASCII above NBSP (a valid IDN host, e.g. é) is + # untouched. + if any(ord(c) <= 0x20 or 0x7F <= ord(c) <= 0xA0 or c in "<>^|" for c in host): + raise ValueError(f"refusing to crawl host with a forbidden character {host!r}") if host == "localhost" or host.endswith(_BLOCKED_SUFFIXES): raise ValueError(f"refusing to crawl local host {host}") try: diff --git a/pkgs/lab-ops-hub/test/test_container_validate.py b/pkgs/lab-ops-hub/test/test_container_validate.py index 2e2cd463a..b7ba8c7b0 100644 --- a/pkgs/lab-ops-hub/test/test_container_validate.py +++ b/pkgs/lab-ops-hub/test/test_container_validate.py @@ -248,6 +248,29 @@ def test_refuses_space_hosts_and_ipvfuture_literals(self): ): self.assertRefused(url) + def test_refuses_del_c1_nbsp_and_forbidden_host_characters(self): + # The #507 check stopped at 0x20, one codepoint short of + # http.client's own set and short of the rest of the WHATWG + # forbidden-host set: DEL, the C1 controls, NBSP and < > ^ | + # still fell through as "a public DNS name", while the + # Worker's WHATWG validator refuses every one (#510). + for url in ( + "http://exam\x7fple.com/", # DEL + "http://exam\x80ple.com/", # C1 control, low end + "http://exam\x85ple.com/", # C1 control (NEL) + "http://exam\x9fple.com/", # C1 control, high end + "http://exam\xa0ple.com/", # NBSP + "http://example.com/", + "http://exam^ple.com/", + "http://exam|ple.com/", + ): + self.assertRefused(url) + # Controls: characters above NBSP and ordinary DNS punctuation + # are not forbidden — valid IDN hosts stay crawlable. + self.assertAccepted("http://exam_ple.com/") + self.assertAccepted("http://examéple.com/") + def test_accepts_public_targets(self): for url in ( "https://example.com/page",