/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package htmltext import ( "io" "net/http" "net/url" "regexp" "strings" "sync/atomic" "unicode" "unicode/utf8" "github.com/Machiel/slugify" "github.com/apache/answer/pkg/checker" "github.com/apache/answer/pkg/converter" strip "github.com/grokify/html-strip-tags-go" "github.com/mozillazg/go-pinyin" "github.com/mozillazg/go-unidecode" ) var ( reCode = regexp.MustCompile(`(?ism)<(pre)>.*<\/pre>`) reCodeReplace = "{code...}" reLink = regexp.MustCompile(`(?ism)(.*)?<\/a>`) reLinkReplace = " [$1] " reSpace = regexp.MustCompile(` +`) reSpaceReplace = " " spaceReplacer = strings.NewReplacer( "\n", " ", "\r", " ", "\t", " ", ) // Without this, pure non-Latin titles (Arabic, Cyrillic, Hebrew, ...) get // stripped by slugify and collapse to the "topic" fallback. Chinese is // handled separately by convertChinese. transliterateNonLatin atomic.Bool ) func init() { transliterateNonLatin.Store(true) } // SetTransliterateNonLatin toggles non-Latin script transliteration for URL slugs. func SetTransliterateNonLatin(enabled bool) { transliterateNonLatin.Store(enabled) } // IsTransliterateNonLatinEnabled reports whether non-Latin transliteration is on. func IsTransliterateNonLatinEnabled() bool { return transliterateNonLatin.Load() } // ClearText clear HTML, get the clear text func ClearText(html string) string { if html == "" { return html } html = reCode.ReplaceAllString(html, reCodeReplace) html = reLink.ReplaceAllString(html, reLinkReplace) text := spaceReplacer.Replace(strip.StripTags(html)) // replace multiple spaces to one space return strings.TrimSpace(reSpace.ReplaceAllString(text, reSpaceReplace)) } func UrlTitle(title string) (text string) { title = convertChinese(title) if transliterateNonLatin.Load() { title = convertNonLatin(title) } title = clearEmoji(title) title = slugify.Slugify(title) title = url.QueryEscape(title) title = cutLongTitle(title) if len(title) == 0 { title = "topic" } return title } func clearEmoji(s string) string { var ret strings.Builder rs := []rune(s) for i := range rs { if len(string(rs[i])) != 4 { ret.WriteString(string(rs[i])) } } return ret.String() } func convertChinese(content string) string { has := checker.IsChinese(content) if !has { return content } return strings.Join(pinyin.LazyConvert(content, nil), "-") } // Short-circuits on Latin-only / Chinese-only input so existing slugs stay byte-identical. func convertNonLatin(content string) string { if !containsNonLatin(content) { return content } return unidecode.Unidecode(content) } func containsNonLatin(content string) bool { for _, r := range content { switch { case r < 0x0080: // ASCII continue case r >= 0x0080 && r <= 0x024F: // Latin-1 Supplement, Latin Extended-A/B continue case unicode.Is(unicode.Han, r): // handled by convertChinese continue case unicode.IsLetter(r): return true } } return false } func cutLongTitle(title string) string { maxBytes := 150 if len(title) <= maxBytes { return title } truncated := title[:maxBytes] for len(truncated) > 0 && !utf8.ValidString(truncated) { truncated = truncated[:len(truncated)-1] } return truncated } // FetchExcerpt return the excerpt from the HTML string func FetchExcerpt(html, trimMarker string, limit int) (text string) { return FetchRangedExcerpt(html, trimMarker, 0, limit) } // findFirstMatchedWord returns the first matched word and its index func findFirstMatchedWord(text string, words []string) (string, int) { if len(text) == 0 || len(words) == 0 { return "", 0 } words = converter.UniqueArray(words) firstWord := "" firstIndex := len(text) for _, word := range words { if idx := strings.Index(text, word); idx != -1 && idx < firstIndex { firstIndex = idx firstWord = word } } if firstIndex != len(text) { return firstWord, firstIndex } return "", 0 } // getRuneRange returns the valid begin and end indexes of the runeText func getRuneRange(runeText []rune, offset, limit int) (begin, end int) { runeLen := len(runeText) limit = min(runeLen, max(0, limit)) begin = min(runeLen, max(0, offset)) end = min(runeLen, begin+limit) return } // FetchRangedExcerpt returns a ranged excerpt from the HTML string. // Note: offset is a rune index, not a byte index func FetchRangedExcerpt(html, trimMarker string, offset int, limit int) (text string) { if len(html) == 0 { text = html return } runeText := []rune(ClearText(html)) begin, end := getRuneRange(runeText, offset, limit) text = string(runeText[begin:end]) if begin > 0 { text = trimMarker + text } if end < len(runeText) { text += trimMarker } return } // FetchMatchedExcerpt returns the matched excerpt according to the words func FetchMatchedExcerpt(html string, words []string, trimMarker string, trimLength int) string { text := ClearText(html) matchedWord, matchedIndex := findFirstMatchedWord(text, words) runeIndex := utf8.RuneCountInString(text[0:matchedIndex]) trimLength = max(0, trimLength) runeOffset := runeIndex - trimLength runeLimit := trimLength + trimLength + utf8.RuneCountInString(matchedWord) textRuneCount := utf8.RuneCountInString(text) if runeOffset+runeLimit > textRuneCount { // Reserved extra chars before the matched word runeOffset = textRuneCount - runeLimit } return FetchRangedExcerpt(html, trimMarker, runeOffset, runeLimit) } func GetPicByUrl(url string) string { res, err := http.Get(url) if err != nil { return "" } defer func() { _ = res.Body.Close() }() pix, err := io.ReadAll(res.Body) if err != nil { return "" } return string(pix) }