Add a meta charset to saved pages so text does not mojibake

A page that declared its charset only in the HTTP Content-Type header, with no meta charset in the markup, lost that signal once kage saved it as a file. A reader serving the bytes without a charset then fell back to its locale encoding and garbled every curly quote, dash, and nbsp. kage writes UTF-8, so it now inserts a meta charset utf-8 at the top of head when the page does not already declare one. Verified on the blog post from the report: the saved HTML now carries the meta and renders the quotes correctly.
Merge pull request #28 from tamnd/release-0.3.1
2026-06-16 00:15:27 +07:00 · 2026-06-15 23:14:41 +07:00
3 changed files with 121 additions and 0 deletions
@@ -6,6 +6,13 @@ All notable changes to kage are recorded here. The format follows

 ## [Unreleased]

+### Fixed
+
+- Saved pages now declare their character encoding, so text no longer mojibakes in a reader.
+  kage writes every page as UTF-8, but a source that set its charset only in the HTTP `Content-Type` header, with no `<meta charset>` in the markup, lost that signal once the page became a standalone file.
+  A reader serving the bytes without a charset then fell back to its locale encoding and turned every curly quote, dash, and non-breaking space into mojibake (reported in #16 and #29).
+  kage now inserts a `<meta charset="utf-8">` at the top of `<head>` when the page does not already declare one, so the page is self-describing in any reader.
+
 ## [0.3.1] - 2026-06-15

 ### Fixed
@@ -41,6 +41,7 @@ type Report struct {
 	MetaRefreshRemoved  int
 	DeadLinksRemoved    int
 	CondCommentsRemoved int
+	CharsetAdded        bool
 }

 // jsURLAttrs are attributes whose value may be a javascript: URL.
@@ -70,6 +71,7 @@ func Strip(doc []byte, opts Options) ([]byte, Report, error) {
 func CleanTree(root *html.Node, opts Options) Report {
 	var rep Report
 	clean(root, opts, &rep)
+	rep.CharsetAdded = ensureCharset(root)
 	if opts.Banner != "" {
 		insertBanner(root, opts.Banner)
 	}
@@ -227,6 +229,68 @@ func unwrapNoscript(parent, ns *html.Node) {
 	parent.RemoveChild(ns)
 }

+// ensureCharset guarantees the document declares UTF-8, inserting a
+// <meta charset="utf-8"> at the top of <head> when none is present, and reports
+// whether it added one. kage renders every saved page as UTF-8, but a source
+// that set its charset only in the HTTP Content-Type header, with no <meta>
+// charset in the markup, loses that signal once the page is a standalone file.
+// A reader then serving the bytes without a charset falls back to its locale
+// encoding and mojibakes every multibyte character (curly quotes, dashes, a
+// non-breaking space). Declaring the charset in the markup makes the page
+// self-describing in any reader, kage's own viewer and Kiwix alike.
+func ensureCharset(root *html.Node) bool {
+	head := findElement(root, atom.Head)
+	if head == nil {
+		return false
+	}
+	if hasCharsetMeta(head) {
+		return false
+	}
+	meta := &html.Node{
+		Type:     html.ElementNode,
+		Data:     "meta",
+		DataAtom: atom.Meta,
+		Attr:     []html.Attribute{{Key: "charset", Val: "utf-8"}},
+	}
+	// The declaration must precede any content for a reader to honour it, so it
+	// goes first in <head>.
+	head.InsertBefore(meta, head.FirstChild)
+	return true
+}
+
+// hasCharsetMeta reports whether head already declares a character encoding,
+// either as <meta charset="..."> or the older <meta http-equiv="Content-Type"
+// content="...; charset=...">.
+func hasCharsetMeta(head *html.Node) bool {
+	for c := head.FirstChild; c != nil; c = c.NextSibling {
+		if c.Type != html.ElementNode || c.DataAtom != atom.Meta {
+			continue
+		}
+		if attr(c, "charset") != "" {
+			return true
+		}
+		if strings.EqualFold(attr(c, "http-equiv"), "content-type") &&
+			strings.Contains(strings.ToLower(attr(c, "content")), "charset=") {
+			return true
+		}
+	}
+	return false
+}
+
+// findElement returns the first element node of the given atom in document
+// order, or nil if none exists.
+func findElement(n *html.Node, a atom.Atom) *html.Node {
+	if n.Type == html.ElementNode && n.DataAtom == a {
+		return n
+	}
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		if found := findElement(c, a); found != nil {
+			return found
+		}
+	}
+	return nil
+}
+
 // insertBanner prepends an HTML comment to the document.
 func insertBanner(root *html.Node, text string) {
 	c := &html.Node{Type: html.CommentNode, Data: " " + text + " "}
@@ -166,3 +166,53 @@ func TestKeepMetaRefreshPlain(t *testing.T) {
 		t.Error("JS-target meta refresh must be removed regardless")
 	}
 }
+
+func TestCharsetAddedWhenMissing(t *testing.T) {
+	// A page whose source declared its charset only in the HTTP header has no
+	// <meta charset>. The saved file must gain one so a reader does not fall back
+	// to its locale encoding and mojibake the UTF-8 text.
+	in := `<html><head><title>Quotes</title></head><body><p>` +
+		"“curly” — café</p></body></html>"
+	out, rep, err := Strip([]byte(in), Options{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !rep.CharsetAdded {
+		t.Error("CharsetAdded = false, want true")
+	}
+	s := string(out)
+	if !strings.Contains(strings.ToLower(s), `<meta charset="utf-8"/>`) {
+		t.Errorf("expected an injected meta charset:\n%s", s)
+	}
+	// It must sit at the very start of <head>, before any content.
+	headIdx := strings.Index(s, "<head>")
+	metaIdx := strings.Index(strings.ToLower(s), "<meta charset")
+	titleIdx := strings.Index(s, "<title>")
+	if headIdx >= metaIdx || metaIdx >= titleIdx {
+		t.Errorf("meta charset must come first in head (head=%d meta=%d title=%d)", headIdx, metaIdx, titleIdx)
+	}
+	// The original bytes are preserved as UTF-8.
+	if !strings.Contains(s, "café") {
+		t.Error("UTF-8 content should be preserved")
+	}
+}
+
+func TestCharsetNotDuplicated(t *testing.T) {
+	// A page that already declares a charset, in either form, is left alone.
+	cases := []string{
+		`<html><head><meta charset="utf-8"><title>x</title></head><body></body></html>`,
+		`<html><head><meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"><title>x</title></head><body></body></html>`,
+	}
+	for _, in := range cases {
+		out, rep, err := Strip([]byte(in), Options{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if rep.CharsetAdded {
+			t.Errorf("CharsetAdded = true for a page that already declares one:\n%s", in)
+		}
+		if n := strings.Count(strings.ToLower(string(out)), "charset"); n != 1 {
+			t.Errorf("charset count = %d, want 1:\n%s", n, out)
+		}
+	}
+}