chore: import upstream snapshot with attribution

pkg/htmltext/htmltext.go

@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package htmltext
+
+import (
+	"io"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"sync/atomic"
+	"unicode"
+	"unicode/utf8"
+
+	"github.com/Machiel/slugify"
+	"github.com/apache/answer/pkg/checker"
+	"github.com/apache/answer/pkg/converter"
+	strip "github.com/grokify/html-strip-tags-go"
+	"github.com/mozillazg/go-pinyin"
+	"github.com/mozillazg/go-unidecode"
+)
+
+var (
+	reCode         = regexp.MustCompile(`(?ism)<(pre)>.*<\/pre>`)
+	reCodeReplace  = "{code...}"
+	reLink         = regexp.MustCompile(`(?ism)<a.*?[^<]>(.*)?<\/a>`)
+	reLinkReplace  = " [$1] "
+	reSpace        = regexp.MustCompile(` +`)
+	reSpaceReplace = " "
+
+	spaceReplacer = strings.NewReplacer(
+		"\n", " ",
+		"\r", " ",
+		"\t", " ",
+	)
+
+	// Without this, pure non-Latin titles (Arabic, Cyrillic, Hebrew, ...) get
+	// stripped by slugify and collapse to the "topic" fallback. Chinese is
+	// handled separately by convertChinese.
+	transliterateNonLatin atomic.Bool
+)
+
+func init() {
+	transliterateNonLatin.Store(true)
+}
+
+// SetTransliterateNonLatin toggles non-Latin script transliteration for URL slugs.
+func SetTransliterateNonLatin(enabled bool) {
+	transliterateNonLatin.Store(enabled)
+}
+
+// IsTransliterateNonLatinEnabled reports whether non-Latin transliteration is on.
+func IsTransliterateNonLatinEnabled() bool {
+	return transliterateNonLatin.Load()
+}
+
+// ClearText clear HTML, get the clear text
+func ClearText(html string) string {
+	if html == "" {
+		return html
+	}
+
+	html = reCode.ReplaceAllString(html, reCodeReplace)
+	html = reLink.ReplaceAllString(html, reLinkReplace)
+
+	text := spaceReplacer.Replace(strip.StripTags(html))
+
+	// replace multiple spaces to one space
+	return strings.TrimSpace(reSpace.ReplaceAllString(text, reSpaceReplace))
+}
+
+func UrlTitle(title string) (text string) {
+	title = convertChinese(title)
+	if transliterateNonLatin.Load() {
+		title = convertNonLatin(title)
+	}
+	title = clearEmoji(title)
+	title = slugify.Slugify(title)
+	title = url.QueryEscape(title)
+	title = cutLongTitle(title)
+	if len(title) == 0 {
+		title = "topic"
+	}
+	return title
+}
+
+func clearEmoji(s string) string {
+	var ret strings.Builder
+	rs := []rune(s)
+	for i := range rs {
+		if len(string(rs[i])) != 4 {
+			ret.WriteString(string(rs[i]))
+		}
+	}
+	return ret.String()
+}
+
+func convertChinese(content string) string {
+	has := checker.IsChinese(content)
+	if !has {
+		return content
+	}
+	return strings.Join(pinyin.LazyConvert(content, nil), "-")
+}
+
+// Short-circuits on Latin-only / Chinese-only input so existing slugs stay byte-identical.
+func convertNonLatin(content string) string {
+	if !containsNonLatin(content) {
+		return content
+	}
+	return unidecode.Unidecode(content)
+}
+
+func containsNonLatin(content string) bool {
+	for _, r := range content {
+		switch {
+		case r < 0x0080: // ASCII
+			continue
+		case r >= 0x0080 && r <= 0x024F: // Latin-1 Supplement, Latin Extended-A/B
+			continue
+		case unicode.Is(unicode.Han, r): // handled by convertChinese
+			continue
+		case unicode.IsLetter(r):
+			return true
+		}
+	}
+	return false
+}
+
+func cutLongTitle(title string) string {
+	maxBytes := 150
+	if len(title) <= maxBytes {
+		return title
+	}
+
+	truncated := title[:maxBytes]
+	for len(truncated) > 0 && !utf8.ValidString(truncated) {
+		truncated = truncated[:len(truncated)-1]
+	}
+	return truncated
+}
+
+// FetchExcerpt return the excerpt from the HTML string
+func FetchExcerpt(html, trimMarker string, limit int) (text string) {
+	return FetchRangedExcerpt(html, trimMarker, 0, limit)
+}
+
+// findFirstMatchedWord returns the first matched word and its index
+func findFirstMatchedWord(text string, words []string) (string, int) {
+	if len(text) == 0 || len(words) == 0 {
+		return "", 0
+	}
+
+	words = converter.UniqueArray(words)
+	firstWord := ""
+	firstIndex := len(text)
+
+	for _, word := range words {
+		if idx := strings.Index(text, word); idx != -1 && idx < firstIndex {
+			firstIndex = idx
+			firstWord = word
+		}
+	}
+
+	if firstIndex != len(text) {
+		return firstWord, firstIndex
+	}
+
+	return "", 0
+}
+
+// getRuneRange returns the valid begin and end indexes of the runeText
+func getRuneRange(runeText []rune, offset, limit int) (begin, end int) {
+	runeLen := len(runeText)
+
+	limit = min(runeLen, max(0, limit))
+	begin = min(runeLen, max(0, offset))
+	end = min(runeLen, begin+limit)
+
+	return
+}
+
+// FetchRangedExcerpt returns a ranged excerpt from the HTML string.
+// Note: offset is a rune index, not a byte index
+func FetchRangedExcerpt(html, trimMarker string, offset int, limit int) (text string) {
+	if len(html) == 0 {
+		text = html
+		return
+	}
+
+	runeText := []rune(ClearText(html))
+	begin, end := getRuneRange(runeText, offset, limit)
+	text = string(runeText[begin:end])
+
+	if begin > 0 {
+		text = trimMarker + text
+	}
+	if end < len(runeText) {
+		text += trimMarker
+	}
+
+	return
+}
+
+// FetchMatchedExcerpt returns the matched excerpt according to the words
+func FetchMatchedExcerpt(html string, words []string, trimMarker string, trimLength int) string {
+	text := ClearText(html)
+	matchedWord, matchedIndex := findFirstMatchedWord(text, words)
+	runeIndex := utf8.RuneCountInString(text[0:matchedIndex])
+
+	trimLength = max(0, trimLength)
+	runeOffset := runeIndex - trimLength
+	runeLimit := trimLength + trimLength + utf8.RuneCountInString(matchedWord)
+
+	textRuneCount := utf8.RuneCountInString(text)
+	if runeOffset+runeLimit > textRuneCount {
+		// Reserved extra chars before the matched word
+		runeOffset = textRuneCount - runeLimit
+	}
+
+	return FetchRangedExcerpt(html, trimMarker, runeOffset, runeLimit)
+}
+
+func GetPicByUrl(url string) string {
+	res, err := http.Get(url)
+	if err != nil {
+		return ""
+	}
+	defer func() {
+		_ = res.Body.Close()
+	}()
+	pix, err := io.ReadAll(res.Body)
+	if err != nil {
+		return ""
+	}
+	return string(pix)
+}