diff --git a/CHANGELOG.md b/CHANGELOG.md index edcd946..6b94bc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,17 @@ All notable changes to kage are recorded here. The format follows thousand are shown separately as "variants", so the live counter tracks the site's real size instead of being inflated by `?q=…` permutations. +### Fixed + +- An asset URL whose query string carries a raw space is now requested with the + space percent-encoded, so the server gets a valid request instead of rejecting + it. Real sites bust a stylesheet's cache with a date, producing an href like + `styles/main.css?Thursday, 26-Feb-2026 16:26:41 UTC`; a browser encodes the + spaces before requesting, but kage was passing them through verbatim and the + server answered `400 Bad Request`. On a developer.apple.com crawl this was the + cause of the large majority of the download errors. The query is re-encoded on + the canonical URL, so the on-disk key matches the fixed request. + ## [0.2.1] - 2026-06-15 ### Added diff --git a/urlx/urlx.go b/urlx/urlx.go index 5c41e44..f0cc1f4 100644 --- a/urlx/urlx.go +++ b/urlx/urlx.go @@ -142,9 +142,59 @@ func canonical(u *url.URL, root bool) *url.URL { } c.Path = cleaned } + c.RawQuery = safeRawQuery(c.RawQuery) return &c } +// safeRawQuery percent-encodes the characters a query string is not allowed to +// carry on the wire (a space, a control byte, a non-ASCII byte) while leaving +// everything already legal untouched, including existing %XX escapes and the +// query sub-delimiters a cache-buster relies on (& = , ; etc.). It exists +// because real sites emit hrefs with raw spaces in the query, e.g. a CSS link +// busted with a date string like "?Thursday, 26-Feb-2026 16:26:41 UTC"; a +// browser encodes the spaces before requesting, but url.Parse keeps RawQuery +// verbatim, so without this the request line is malformed and the server +// answers 400. Re-encoding here, on the canonical URL, fixes both the fetch and +// the on-disk key in one place. +func safeRawQuery(raw string) string { + if raw == "" { + return "" + } + var b strings.Builder + for i := 0; i < len(raw); i++ { + ch := raw[i] + switch { + case ch == '%' && i+2 < len(raw) && isHex(raw[i+1]) && isHex(raw[i+2]): + // Already a percent-escape; copy it through unchanged. + b.WriteString(raw[i : i+3]) + i += 2 + case queryByteAllowed(ch): + b.WriteByte(ch) + default: + b.WriteByte('%') + const hexDigits = "0123456789ABCDEF" + b.WriteByte(hexDigits[ch>>4]) + b.WriteByte(hexDigits[ch&0x0f]) + } + } + return b.String() +} + +// queryByteAllowed reports whether ch may appear literally in a URL query. +// It is the RFC 3986 query grammar (pchar / "/" / "?"), which keeps the +// sub-delims that structure a query and rejects spaces, quotes, and controls. +func queryByteAllowed(ch byte) bool { + switch { + case ch >= 'a' && ch <= 'z', ch >= 'A' && ch <= 'Z', ch >= '0' && ch <= '9': + return true + } + return strings.IndexByte("-._~!$&'()*+,;=:@/?", ch) >= 0 +} + +func isHex(ch byte) bool { + return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F') +} + // Key is the canonical string form used to dedup pages and assets. func Key(u *url.URL) string { return u.String() } diff --git a/urlx/urlx_test.go b/urlx/urlx_test.go index 87b6353..cc78a38 100644 --- a/urlx/urlx_test.go +++ b/urlx/urlx_test.go @@ -61,6 +61,12 @@ func TestNormalize(t *testing.T) { {"//cdn.io/x.css", "https://cdn.io/x.css", false}, {"HTTPS://Other.COM/Y", "https://other.com/Y", false}, {"sub/", "https://ex.com/docs/sub/", false}, + // A query busted with a raw date string must come back with its spaces + // percent-encoded so the request line is valid, while the commas and + // colons a query legally carries stay as they are. + {"a.css?Thursday, 26-Feb-2026 16:26:41 UTC", "https://ex.com/docs/a.css?Thursday,%2026-Feb-2026%2016:26:41%20UTC", false}, + {"b.css?v=1&t=a b", "https://ex.com/docs/b.css?v=1&t=a%20b", false}, + {"c.css?x=%20done", "https://ex.com/docs/c.css?x=%20done", false}, {"#top", "", true}, {"javascript:void(0)", "", true}, {"mailto:a@b.com", "", true},