package sanitize import ( "strings" "sync" "unicode" "github.com/microcosm-cc/bluemonday" ) var policy *bluemonday.Policy var policyOnce sync.Once func Sanitize(input string) string { return FilterHTMLTags(FilterCodeFenceMetadata(FilterInvisibleCharacters(input))) } // FilterInvisibleCharacters removes invisible or control characters that should not appear // in user-facing titles or bodies. This includes: // - Unicode tag characters: U+E0001, U+E0020–U+E007F // - BiDi control characters: U+202A–U+202E, U+2066–U+2069 // - Hidden modifier characters: U+200B, U+200C, U+200E, U+200F, U+00AD, U+FEFF, U+180E, U+2060–U+2064 func FilterInvisibleCharacters(input string) string { if input == "" { return input } // Filter runes out := make([]rune, 0, len(input)) for _, r := range input { if !shouldRemoveRune(r) { out = append(out, r) } } return string(out) } func FilterHTMLTags(input string) string { if input == "" { return input } return getPolicy().Sanitize(input) } // FilterCodeFenceMetadata removes hidden or suspicious info strings from fenced code blocks. func FilterCodeFenceMetadata(input string) string { if input == "" { return input } lines := strings.Split(input, "\n") insideFence := false currentFenceLen := 0 for i, line := range lines { sanitized, toggled, fenceLen := sanitizeCodeFenceLine(line, insideFence, currentFenceLen) lines[i] = sanitized if toggled { insideFence = !insideFence if insideFence { currentFenceLen = fenceLen } else { currentFenceLen = 0 } } } return strings.Join(lines, "\n") } const maxCodeFenceInfoLength = 48 func sanitizeCodeFenceLine(line string, insideFence bool, expectedFenceLen int) (string, bool, int) { idx := strings.Index(line, "```") if idx == -1 { return line, false, expectedFenceLen } if hasNonWhitespace(line[:idx]) { return line, false, expectedFenceLen } fenceEnd := idx for fenceEnd < len(line) && line[fenceEnd] == '`' { fenceEnd++ } fenceLen := fenceEnd - idx if fenceLen < 3 { return line, false, expectedFenceLen } rest := line[fenceEnd:] if insideFence { if expectedFenceLen != 0 && fenceLen != expectedFenceLen { return line, false, expectedFenceLen } return line[:fenceEnd], true, fenceLen } trimmed := strings.TrimSpace(rest) if trimmed == "" { return line[:fenceEnd], true, fenceLen } if strings.IndexFunc(trimmed, unicode.IsSpace) != -1 { return line[:fenceEnd], true, fenceLen } if len(trimmed) > maxCodeFenceInfoLength { return line[:fenceEnd], true, fenceLen } if !isSafeCodeFenceToken(trimmed) { return line[:fenceEnd], true, fenceLen } if len(rest) > 0 && unicode.IsSpace(rune(rest[0])) { return line[:fenceEnd] + " " + trimmed, true, fenceLen } return line[:fenceEnd] + trimmed, true, fenceLen } func hasNonWhitespace(segment string) bool { for _, r := range segment { if !unicode.IsSpace(r) { return true } } return false } func isSafeCodeFenceToken(token string) bool { for _, r := range token { if unicode.IsLetter(r) || unicode.IsDigit(r) { continue } switch r { case '+', '-', '_', '#', '.': continue } return false } return true } func getPolicy() *bluemonday.Policy { policyOnce.Do(func() { p := bluemonday.StrictPolicy() p.AllowElements( "b", "blockquote", "br", "code", "em", "h1", "h2", "h3", "h4", "h5", "h6", "hr", "i", "li", "ol", "p", "pre", "strong", "sub", "sup", "table", "tbody", "td", "th", "thead", "tr", "ul", "a", "img", ) p.AllowAttrs("href").OnElements("a") p.AllowURLSchemes("http", "https") p.RequireParseableURLs(true) p.RequireNoFollowOnLinks(true) p.RequireNoReferrerOnLinks(true) p.AddTargetBlankToFullyQualifiedLinks(true) p.AllowImages() p.AllowAttrs("src", "alt", "title").OnElements("img") policy = p }) return policy } func shouldRemoveRune(r rune) bool { switch r { case 0x200B, // ZERO WIDTH SPACE 0x200C, // ZERO WIDTH NON-JOINER 0x200E, // LEFT-TO-RIGHT MARK 0x200F, // RIGHT-TO-LEFT MARK 0x00AD, // SOFT HYPHEN 0xFEFF, // ZERO WIDTH NO-BREAK SPACE 0x180E: // MONGOLIAN VOWEL SEPARATOR return true case 0xE0001: // TAG return true } // Ranges // Unicode tags: U+E0020–U+E007F if r >= 0xE0020 && r <= 0xE007F { return true } // BiDi controls: U+202A–U+202E if r >= 0x202A && r <= 0x202E { return true } // BiDi isolates: U+2066–U+2069 if r >= 0x2066 && r <= 0x2069 { return true } // Hidden modifiers: U+2060–U+2064 if r >= 0x2060 && r <= 0x2064 { return true } return false }