Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions pkgs/lab-ops-hub/container/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,12 @@ def validate_crawl_url(raw):
# the zone id). Refuse every % host, fail-closed (#505).
if "%" in host:
raise ValueError(f"refusing to crawl percent-encoded host {host}")
# A space or control character in the host is not a DNS name: WHATWG
# rejects it outright, urlparse hands it through verbatim, and
# http.client only raises for it deep inside the fetcher — the same
# lazy-failure shape #502 fixed for ports. Refuse up front (#507).
if any(ord(c) <= 0x20 for c in host):

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Reject DEL alongside the other control characters

For a target such as http://exam\x7fple.com/, urlparse preserves DEL in the hostname, but this predicate accepts it because U+007F is above U+0020. The Worker’s WHATWG parser rejects the URL, while the container proceeds until http.client raises InvalidURL, recreating the deep-fetch failure this change is intended to prevent. Include DEL (and preferably the full relevant control-character range) in the upfront rejection.

Useful? React with 👍 / 👎.

raise ValueError(f"refusing to crawl host with whitespace {host!r}")
if host == "localhost" or host.endswith(_BLOCKED_SUFFIXES):
raise ValueError(f"refusing to crawl local host {host}")
try:
Expand All @@ -76,6 +82,17 @@ def validate_crawl_url(raw):
f"refusing to crawl host {host}: numeric final label "
"but not a valid IPv4 address"
)
# A bracketed host occupied the IP-literal position
# ([v1.fe80::], [v7.a] and other IPvFuture forms) but is not
# a valid IPv6 literal — urlparse strips the brackets, so
# look at the netloc. Fail closed instead of reclassifying
# it as a public DNS name, matching the Worker's validator
# (#507).
if u.netloc.rsplit("@", 1)[-1].startswith("["):
raise ValueError(
f"refusing to crawl host {host}: bracketed literal "
"but not a valid IPv6 address"
)
return raw # a public DNS name; literal-IP policy does not apply
embedded = _embedded_ipv4(ip)
if embedded is not None:
Expand Down
16 changes: 16 additions & 0 deletions pkgs/lab-ops-hub/test/test_container_validate.py
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,22 @@ def test_refuses_percent_encoded_hosts_and_zone_ids(self):
):
self.assertRefused(url)

def test_refuses_space_hosts_and_ipvfuture_literals(self):
# A space in the host is not a DNS name: WHATWG rejects it and
# http.client only raises for it deep inside the fetcher, so the
# validator refuses up front. A bracketed host that is not a
# valid IPv6 literal (IPvFuture forms) occupied the IP-literal
# position; a DNS name never contains a colon, so it must not be
# reclassified as a public DNS name either (#507).
for url in (
"http://exam ple.com/",
"http://exa mple.com/",
"http://[v1.fe80::]/",
"http://[v7.a]/",
"http://[vf.FF]/",
):
self.assertRefused(url)

def test_accepts_public_targets(self):
for url in (
"https://example.com/page",
Expand Down
Loading