diff --git a/CHANGELOG.md b/CHANGELOG.md
index e25e6b5..65c6dd2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,13 @@ All notable changes to kage are recorded here. The format follows
## [Unreleased]
+### Fixed
+
+- Saved pages now declare their character encoding, so text no longer mojibakes in a reader.
+ kage writes every page as UTF-8, but a source that set its charset only in the HTTP `Content-Type` header, with no `` in the markup, lost that signal once the page became a standalone file.
+ A reader serving the bytes without a charset then fell back to its locale encoding and turned every curly quote, dash, and non-breaking space into mojibake (reported in #16 and #29).
+ kage now inserts a `` at the top of `` when the page does not already declare one, so the page is self-describing in any reader.
+
## [0.3.1] - 2026-06-15
### Fixed
diff --git a/sanitize/sanitize.go b/sanitize/sanitize.go
index 544458a..cc9202f 100644
--- a/sanitize/sanitize.go
+++ b/sanitize/sanitize.go
@@ -41,6 +41,7 @@ type Report struct {
MetaRefreshRemoved int
DeadLinksRemoved int
CondCommentsRemoved int
+ CharsetAdded bool
}
// jsURLAttrs are attributes whose value may be a javascript: URL.
@@ -70,6 +71,7 @@ func Strip(doc []byte, opts Options) ([]byte, Report, error) {
func CleanTree(root *html.Node, opts Options) Report {
var rep Report
clean(root, opts, &rep)
+ rep.CharsetAdded = ensureCharset(root)
if opts.Banner != "" {
insertBanner(root, opts.Banner)
}
@@ -227,6 +229,68 @@ func unwrapNoscript(parent, ns *html.Node) {
parent.RemoveChild(ns)
}
+// ensureCharset guarantees the document declares UTF-8, inserting a
+// at the top of when none is present, and reports
+// whether it added one. kage renders every saved page as UTF-8, but a source
+// that set its charset only in the HTTP Content-Type header, with no
+// charset in the markup, loses that signal once the page is a standalone file.
+// A reader then serving the bytes without a charset falls back to its locale
+// encoding and mojibakes every multibyte character (curly quotes, dashes, a
+// non-breaking space). Declaring the charset in the markup makes the page
+// self-describing in any reader, kage's own viewer and Kiwix alike.
+func ensureCharset(root *html.Node) bool {
+ head := findElement(root, atom.Head)
+ if head == nil {
+ return false
+ }
+ if hasCharsetMeta(head) {
+ return false
+ }
+ meta := &html.Node{
+ Type: html.ElementNode,
+ Data: "meta",
+ DataAtom: atom.Meta,
+ Attr: []html.Attribute{{Key: "charset", Val: "utf-8"}},
+ }
+ // The declaration must precede any content for a reader to honour it, so it
+ // goes first in .
+ head.InsertBefore(meta, head.FirstChild)
+ return true
+}
+
+// hasCharsetMeta reports whether head already declares a character encoding,
+// either as or the older .
+func hasCharsetMeta(head *html.Node) bool {
+ for c := head.FirstChild; c != nil; c = c.NextSibling {
+ if c.Type != html.ElementNode || c.DataAtom != atom.Meta {
+ continue
+ }
+ if attr(c, "charset") != "" {
+ return true
+ }
+ if strings.EqualFold(attr(c, "http-equiv"), "content-type") &&
+ strings.Contains(strings.ToLower(attr(c, "content")), "charset=") {
+ return true
+ }
+ }
+ return false
+}
+
+// findElement returns the first element node of the given atom in document
+// order, or nil if none exists.
+func findElement(n *html.Node, a atom.Atom) *html.Node {
+ if n.Type == html.ElementNode && n.DataAtom == a {
+ return n
+ }
+ for c := n.FirstChild; c != nil; c = c.NextSibling {
+ if found := findElement(c, a); found != nil {
+ return found
+ }
+ }
+ return nil
+}
+
// insertBanner prepends an HTML comment to the document.
func insertBanner(root *html.Node, text string) {
c := &html.Node{Type: html.CommentNode, Data: " " + text + " "}
diff --git a/sanitize/sanitize_test.go b/sanitize/sanitize_test.go
index f663a12..0e90595 100644
--- a/sanitize/sanitize_test.go
+++ b/sanitize/sanitize_test.go
@@ -166,3 +166,53 @@ func TestKeepMetaRefreshPlain(t *testing.T) {
t.Error("JS-target meta refresh must be removed regardless")
}
}
+
+func TestCharsetAddedWhenMissing(t *testing.T) {
+ // A page whose source declared its charset only in the HTTP header has no
+ // . The saved file must gain one so a reader does not fall back
+ // to its locale encoding and mojibake the UTF-8 text.
+ in := `Quotes` +
+ "“curly” — café
"
+ out, rep, err := Strip([]byte(in), Options{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !rep.CharsetAdded {
+ t.Error("CharsetAdded = false, want true")
+ }
+ s := string(out)
+ if !strings.Contains(strings.ToLower(s), ``) {
+ t.Errorf("expected an injected meta charset:\n%s", s)
+ }
+ // It must sit at the very start of , before any content.
+ headIdx := strings.Index(s, "")
+ metaIdx := strings.Index(strings.ToLower(s), "")
+ if headIdx >= metaIdx || metaIdx >= titleIdx {
+ t.Errorf("meta charset must come first in head (head=%d meta=%d title=%d)", headIdx, metaIdx, titleIdx)
+ }
+ // The original bytes are preserved as UTF-8.
+ if !strings.Contains(s, "café") {
+ t.Error("UTF-8 content should be preserved")
+ }
+}
+
+func TestCharsetNotDuplicated(t *testing.T) {
+ // A page that already declares a charset, in either form, is left alone.
+ cases := []string{
+ `x`,
+ `x`,
+ }
+ for _, in := range cases {
+ out, rep, err := Strip([]byte(in), Options{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if rep.CharsetAdded {
+ t.Errorf("CharsetAdded = true for a page that already declares one:\n%s", in)
+ }
+ if n := strings.Count(strings.ToLower(string(out)), "charset"); n != 1 {
+ t.Errorf("charset count = %d, want 1:\n%s", n, out)
+ }
+ }
+}