wehub-resource-sync
92 lines
2.3 KiB
Go
92 lines
2.3 KiB
Go
package playlist
|
||
|
||
import (
|
||
"unicode"
|
||
"unicode/utf8"
|
||
|
||
"golang.org/x/text/encoding"
|
||
"golang.org/x/text/encoding/charmap"
|
||
)
|
||
|
||
// legacyEncodings lists codepages commonly used by music taggers that wrote
|
||
// non-Latin text but marked the encoding as Latin-1.
|
||
var legacyEncodings = []encoding.Encoding{
|
||
charmap.Windows1255, // Hebrew
|
||
charmap.Windows1256, // Arabic
|
||
charmap.Windows1251, // Cyrillic
|
||
charmap.Windows1253, // Greek
|
||
charmap.Windows874, // Thai
|
||
}
|
||
|
||
// sanitizeTag detects mojibake from legacy codepages and re-decodes to UTF-8.
|
||
//
|
||
// Many old ID3 taggers write non-Latin text using a Windows codepage but mark
|
||
// the encoding as Latin-1. The tag library faithfully decodes those bytes as
|
||
// Latin-1, producing garbled text. We detect this by checking for a high
|
||
// density of Latin-1 supplement characters (U+0080–U+00FF), reverse the
|
||
// Latin-1 decode to recover the original bytes, then try common codepages
|
||
// and pick the one that produces the most non-Latin script characters.
|
||
func sanitizeTag(s string) string {
|
||
if s == "" {
|
||
return s
|
||
}
|
||
|
||
// Count runes in the Latin-1 supplement range (U+0080–U+00FF).
|
||
// Real Latin text (French, German) rarely exceeds ~10% accented chars.
|
||
// Mojibake from non-Latin scripts is almost entirely in this range.
|
||
var total, highCount int
|
||
for _, r := range s {
|
||
total++
|
||
if r >= 0x80 && r <= 0xFF {
|
||
highCount++
|
||
}
|
||
}
|
||
if total == 0 || highCount*3 < total {
|
||
return s
|
||
}
|
||
|
||
// Reverse the Latin-1 decode to recover the original tag bytes.
|
||
raw := make([]byte, 0, total)
|
||
for _, r := range s {
|
||
if r > 0xFF {
|
||
return s // Contains runes outside Latin-1 — not simple mojibake
|
||
}
|
||
raw = append(raw, byte(r))
|
||
}
|
||
|
||
// The original bytes might be valid UTF-8 that was double-decoded.
|
||
if utf8.Valid(raw) {
|
||
return string(raw)
|
||
}
|
||
|
||
// Try legacy codepages; pick the one that produces the most
|
||
// non-Latin letters (Hebrew, Cyrillic, Arabic, Greek, Thai, etc.).
|
||
var bestText string
|
||
var bestScore int
|
||
for _, enc := range legacyEncodings {
|
||
decoded, err := enc.NewDecoder().Bytes(raw)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
text := string(decoded)
|
||
if !utf8.ValidString(text) {
|
||
continue
|
||
}
|
||
score := 0
|
||
for _, r := range text {
|
||
if unicode.IsLetter(r) && r > 0x024F {
|
||
score++
|
||
}
|
||
}
|
||
if score > bestScore {
|
||
bestScore = score
|
||
bestText = text
|
||
}
|
||
}
|
||
|
||
if bestText != "" {
|
||
return bestText
|
||
}
|
||
return s
|
||
}
|