Percent-encode raw spaces in asset query strings (#24)
A site busts a stylesheet's cache with a date string, so a page links to styles/main.css?Thursday, 26-Feb-2026 16:26:41 UTC. A browser encodes the spaces before requesting, but kage parsed the href and passed RawQuery through verbatim, so the request line carried literal spaces and the server answered 400 Bad Request. On a developer.apple.com crawl this was the bulk of the download errors. Re-encode the query on the canonical URL: percent-encode any byte a query may not carry (space, control, non-ASCII) while leaving valid sub-delims and existing %XX escapes alone. Doing it in canonical fixes the fetch and keeps the on-disk key in step with the request.
This commit is contained in:
@@ -39,6 +39,17 @@ All notable changes to kage are recorded here. The format follows
|
||||
thousand are shown separately as "variants", so the live counter tracks the
|
||||
site's real size instead of being inflated by `?q=…` permutations.
|
||||
|
||||
### Fixed
|
||||
|
||||
- An asset URL whose query string carries a raw space is now requested with the
|
||||
space percent-encoded, so the server gets a valid request instead of rejecting
|
||||
it. Real sites bust a stylesheet's cache with a date, producing an href like
|
||||
`styles/main.css?Thursday, 26-Feb-2026 16:26:41 UTC`; a browser encodes the
|
||||
spaces before requesting, but kage was passing them through verbatim and the
|
||||
server answered `400 Bad Request`. On a developer.apple.com crawl this was the
|
||||
cause of the large majority of the download errors. The query is re-encoded on
|
||||
the canonical URL, so the on-disk key matches the fixed request.
|
||||
|
||||
## [0.2.1] - 2026-06-15
|
||||
|
||||
### Added
|
||||
|
||||
@@ -142,9 +142,59 @@ func canonical(u *url.URL, root bool) *url.URL {
|
||||
}
|
||||
c.Path = cleaned
|
||||
}
|
||||
c.RawQuery = safeRawQuery(c.RawQuery)
|
||||
return &c
|
||||
}
|
||||
|
||||
// safeRawQuery percent-encodes the characters a query string is not allowed to
|
||||
// carry on the wire (a space, a control byte, a non-ASCII byte) while leaving
|
||||
// everything already legal untouched, including existing %XX escapes and the
|
||||
// query sub-delimiters a cache-buster relies on (& = , ; etc.). It exists
|
||||
// because real sites emit hrefs with raw spaces in the query, e.g. a CSS link
|
||||
// busted with a date string like "?Thursday, 26-Feb-2026 16:26:41 UTC"; a
|
||||
// browser encodes the spaces before requesting, but url.Parse keeps RawQuery
|
||||
// verbatim, so without this the request line is malformed and the server
|
||||
// answers 400. Re-encoding here, on the canonical URL, fixes both the fetch and
|
||||
// the on-disk key in one place.
|
||||
func safeRawQuery(raw string) string {
|
||||
if raw == "" {
|
||||
return ""
|
||||
}
|
||||
var b strings.Builder
|
||||
for i := 0; i < len(raw); i++ {
|
||||
ch := raw[i]
|
||||
switch {
|
||||
case ch == '%' && i+2 < len(raw) && isHex(raw[i+1]) && isHex(raw[i+2]):
|
||||
// Already a percent-escape; copy it through unchanged.
|
||||
b.WriteString(raw[i : i+3])
|
||||
i += 2
|
||||
case queryByteAllowed(ch):
|
||||
b.WriteByte(ch)
|
||||
default:
|
||||
b.WriteByte('%')
|
||||
const hexDigits = "0123456789ABCDEF"
|
||||
b.WriteByte(hexDigits[ch>>4])
|
||||
b.WriteByte(hexDigits[ch&0x0f])
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// queryByteAllowed reports whether ch may appear literally in a URL query.
|
||||
// It is the RFC 3986 query grammar (pchar / "/" / "?"), which keeps the
|
||||
// sub-delims that structure a query and rejects spaces, quotes, and controls.
|
||||
func queryByteAllowed(ch byte) bool {
|
||||
switch {
|
||||
case ch >= 'a' && ch <= 'z', ch >= 'A' && ch <= 'Z', ch >= '0' && ch <= '9':
|
||||
return true
|
||||
}
|
||||
return strings.IndexByte("-._~!$&'()*+,;=:@/?", ch) >= 0
|
||||
}
|
||||
|
||||
func isHex(ch byte) bool {
|
||||
return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F')
|
||||
}
|
||||
|
||||
// Key is the canonical string form used to dedup pages and assets.
|
||||
func Key(u *url.URL) string { return u.String() }
|
||||
|
||||
|
||||
@@ -61,6 +61,12 @@ func TestNormalize(t *testing.T) {
|
||||
{"//cdn.io/x.css", "https://cdn.io/x.css", false},
|
||||
{"HTTPS://Other.COM/Y", "https://other.com/Y", false},
|
||||
{"sub/", "https://ex.com/docs/sub/", false},
|
||||
// A query busted with a raw date string must come back with its spaces
|
||||
// percent-encoded so the request line is valid, while the commas and
|
||||
// colons a query legally carries stay as they are.
|
||||
{"a.css?Thursday, 26-Feb-2026 16:26:41 UTC", "https://ex.com/docs/a.css?Thursday,%2026-Feb-2026%2016:26:41%20UTC", false},
|
||||
{"b.css?v=1&t=a b", "https://ex.com/docs/b.css?v=1&t=a%20b", false},
|
||||
{"c.css?x=%20done", "https://ex.com/docs/c.css?x=%20done", false},
|
||||
{"#top", "", true},
|
||||
{"javascript:void(0)", "", true},
|
||||
{"mailto:a@b.com", "", true},
|
||||
|
||||
Reference in New Issue
Block a user