From 16c9fcbcfecb44b335dafa26a96c329272d225a1 Mon Sep 17 00:00:00 2001 From: Duc-Tam Nguyen Date: Tue, 16 Jun 2026 00:09:48 +0700 Subject: [PATCH] Add a meta charset to saved pages so text does not mojibake A page that declared its charset only in the HTTP Content-Type header, with no meta charset in the markup, lost that signal once kage saved it as a file. A reader serving the bytes without a charset then fell back to its locale encoding and garbled every curly quote, dash, and nbsp. kage writes UTF-8, so it now inserts a meta charset utf-8 at the top of head when the page does not already declare one. Verified on the blog post from the report: the saved HTML now carries the meta and renders the quotes correctly. --- CHANGELOG.md | 7 +++++ sanitize/sanitize.go | 64 +++++++++++++++++++++++++++++++++++++++ sanitize/sanitize_test.go | 50 ++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e25e6b5..65c6dd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ All notable changes to kage are recorded here. The format follows ## [Unreleased] +### Fixed + +- Saved pages now declare their character encoding, so text no longer mojibakes in a reader. + kage writes every page as UTF-8, but a source that set its charset only in the HTTP `Content-Type` header, with no `` in the markup, lost that signal once the page became a standalone file. + A reader serving the bytes without a charset then fell back to its locale encoding and turned every curly quote, dash, and non-breaking space into mojibake (reported in #16 and #29). + kage now inserts a `` at the top of `` when the page does not already declare one, so the page is self-describing in any reader. + ## [0.3.1] - 2026-06-15 ### Fixed diff --git a/sanitize/sanitize.go b/sanitize/sanitize.go index 544458a..cc9202f 100644 --- a/sanitize/sanitize.go +++ b/sanitize/sanitize.go @@ -41,6 +41,7 @@ type Report struct { MetaRefreshRemoved int DeadLinksRemoved int CondCommentsRemoved int + CharsetAdded bool } // jsURLAttrs are attributes whose value may be a javascript: URL. @@ -70,6 +71,7 @@ func Strip(doc []byte, opts Options) ([]byte, Report, error) { func CleanTree(root *html.Node, opts Options) Report { var rep Report clean(root, opts, &rep) + rep.CharsetAdded = ensureCharset(root) if opts.Banner != "" { insertBanner(root, opts.Banner) } @@ -227,6 +229,68 @@ func unwrapNoscript(parent, ns *html.Node) { parent.RemoveChild(ns) } +// ensureCharset guarantees the document declares UTF-8, inserting a +// at the top of when none is present, and reports +// whether it added one. kage renders every saved page as UTF-8, but a source +// that set its charset only in the HTTP Content-Type header, with no +// charset in the markup, loses that signal once the page is a standalone file. +// A reader then serving the bytes without a charset falls back to its locale +// encoding and mojibakes every multibyte character (curly quotes, dashes, a +// non-breaking space). Declaring the charset in the markup makes the page +// self-describing in any reader, kage's own viewer and Kiwix alike. +func ensureCharset(root *html.Node) bool { + head := findElement(root, atom.Head) + if head == nil { + return false + } + if hasCharsetMeta(head) { + return false + } + meta := &html.Node{ + Type: html.ElementNode, + Data: "meta", + DataAtom: atom.Meta, + Attr: []html.Attribute{{Key: "charset", Val: "utf-8"}}, + } + // The declaration must precede any content for a reader to honour it, so it + // goes first in . + head.InsertBefore(meta, head.FirstChild) + return true +} + +// hasCharsetMeta reports whether head already declares a character encoding, +// either as or the older . +func hasCharsetMeta(head *html.Node) bool { + for c := head.FirstChild; c != nil; c = c.NextSibling { + if c.Type != html.ElementNode || c.DataAtom != atom.Meta { + continue + } + if attr(c, "charset") != "" { + return true + } + if strings.EqualFold(attr(c, "http-equiv"), "content-type") && + strings.Contains(strings.ToLower(attr(c, "content")), "charset=") { + return true + } + } + return false +} + +// findElement returns the first element node of the given atom in document +// order, or nil if none exists. +func findElement(n *html.Node, a atom.Atom) *html.Node { + if n.Type == html.ElementNode && n.DataAtom == a { + return n + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + if found := findElement(c, a); found != nil { + return found + } + } + return nil +} + // insertBanner prepends an HTML comment to the document. func insertBanner(root *html.Node, text string) { c := &html.Node{Type: html.CommentNode, Data: " " + text + " "} diff --git a/sanitize/sanitize_test.go b/sanitize/sanitize_test.go index f663a12..0e90595 100644 --- a/sanitize/sanitize_test.go +++ b/sanitize/sanitize_test.go @@ -166,3 +166,53 @@ func TestKeepMetaRefreshPlain(t *testing.T) { t.Error("JS-target meta refresh must be removed regardless") } } + +func TestCharsetAddedWhenMissing(t *testing.T) { + // A page whose source declared its charset only in the HTTP header has no + // . The saved file must gain one so a reader does not fall back + // to its locale encoding and mojibake the UTF-8 text. + in := `Quotes

` + + "“curly” — café

" + out, rep, err := Strip([]byte(in), Options{}) + if err != nil { + t.Fatal(err) + } + if !rep.CharsetAdded { + t.Error("CharsetAdded = false, want true") + } + s := string(out) + if !strings.Contains(strings.ToLower(s), ``) { + t.Errorf("expected an injected meta charset:\n%s", s) + } + // It must sit at the very start of , before any content. + headIdx := strings.Index(s, "") + metaIdx := strings.Index(strings.ToLower(s), "") + if headIdx >= metaIdx || metaIdx >= titleIdx { + t.Errorf("meta charset must come first in head (head=%d meta=%d title=%d)", headIdx, metaIdx, titleIdx) + } + // The original bytes are preserved as UTF-8. + if !strings.Contains(s, "café") { + t.Error("UTF-8 content should be preserved") + } +} + +func TestCharsetNotDuplicated(t *testing.T) { + // A page that already declares a charset, in either form, is left alone. + cases := []string{ + `x`, + `x`, + } + for _, in := range cases { + out, rep, err := Strip([]byte(in), Options{}) + if err != nil { + t.Fatal(err) + } + if rep.CharsetAdded { + t.Errorf("CharsetAdded = true for a page that already declares one:\n%s", in) + } + if n := strings.Count(strings.ToLower(string(out)), "charset"); n != 1 { + t.Errorf("charset count = %d, want 1:\n%s", n, out) + } + } +}