/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package htmltext import ( "fmt" "strings" "testing" "github.com/stretchr/testify/assert" ) func TestClearText(t *testing.T) { var ( expected, clearedText string ) // test code clear text expected = "hello{code...}" clearedText = ClearText("

hello

var a = \"good\"

") assert.Equal(t, expected, clearedText) // test link clear text expected = "hello [example.com]" clearedText = ClearText("

hello example.com

") assert.Equal(t, expected, clearedText) clearedText = ClearText("

helloexample.com

") assert.Equal(t, expected, clearedText) expected = "hello world" clearedText = ClearText("
hello
\n
world
") assert.Equal(t, expected, clearedText) } func TestFetchExcerpt(t *testing.T) { var ( expected, text string ) // test english string expected = "hello..." text = FetchExcerpt("

hello world

", "...", 5) assert.Equal(t, expected, text) // test mixed string expected = "hello你好..." text = FetchExcerpt("

hello你好world

", "...", 7) assert.Equal(t, expected, text) // test mixed string with emoticon expected = "hello你好😂..." text = FetchExcerpt("

hello你好😂world

", "...", 8) assert.Equal(t, expected, text) expected = "hello你好" text = FetchExcerpt("

hello你好

", "...", 8) assert.Equal(t, expected, text) } func TestUrlTitle(t *testing.T) { list := []string{ "hello你好😂...", "这是一个,标题,title", } for _, title := range list { formatTitle := UrlTitle(title) fmt.Println(formatTitle) } } func TestUrlTitleTable(t *testing.T) { // Long pure-Arabic title: 50 copies of the same Arabic word, joined by spaces. // Unidecode of "كيف" is "kyf", so the slug becomes "kyf-" repeated and // exceeds cutLongTitle's 150-byte cap. longArabic := strings.Repeat("كيف ", 50) wantLongArabic := strings.Repeat("kyf-", 37) + "ky" // 37*4 + 2 = 150 bytes cases := []struct { name string title string want string }{ { name: "empty", title: "", want: "topic", }, { name: "pure latin unchanged", title: "hello world", want: "hello-world", }, { // Pinyin conversion drops Latin runes by design — matches pre-fix behavior. name: "pure chinese unchanged", title: "这是一个,标题,title", want: "zhe-shi-yi-ge-biao-ti", }, { // The fix: previously collapsed to "topic" for all of these scripts. // Outputs are an ASCII approximation, not linguistically correct // romanization — see PR description. name: "arabic transliterated", title: "كيف حالك", want: "kyf-hlk", }, { name: "mixed latin and arabic", title: "مرحبا hello", want: "mrhb-hello", }, { name: "thai transliterated", title: "ไทย ไทย", want: "aithy-aithy", }, { name: "japanese hiragana transliterated", title: "こんにちは", want: "konnichiha", }, { // Japanese with Han-block kanji is caught by the pre-existing pinyin // pre-step (Chinese reading, not Japanese), so this path is unchanged // by this PR. Pinning to document the existing behavior. name: "japanese kanji goes through pinyin path unchanged", title: "日本", want: "ri-ben", }, { name: "korean transliterated", title: "안녕하세요", want: "annyeonghaseyo", }, { name: "hebrew transliterated", title: "שלום עולם", want: "shlvm-vlm", }, { name: "cyrillic transliterated", title: "Привет мир", want: "privet-mir", }, { name: "emoji only falls back to topic", title: "😂😂😂", want: "topic", }, { name: "long arabic truncates at cutLongTitle boundary", title: longArabic, want: wantLongArabic, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { got := UrlTitle(tc.title) assert.Equal(t, tc.want, got) }) } } func TestUrlTitleTransliterationToggle(t *testing.T) { defer SetTransliterateNonLatin(true) SetTransliterateNonLatin(false) // With transliteration off, pure-Arabic titles collapse to the existing // "topic" fallback (the pre-fix behavior). assert.Equal(t, "topic", UrlTitle("كيف حالك")) SetTransliterateNonLatin(true) assert.Equal(t, "kyf-hlk", UrlTitle("كيف حالك")) } func TestFindFirstMatchedWord(t *testing.T) { var ( expectedWord, actualWord string expectedIndex, actualIndex int ) text := "Hello, I have 中文 and 😂 and I am supposed to work fine." // test find nothing expectedWord, expectedIndex = "", 0 actualWord, actualIndex = findFirstMatchedWord(text, []string{"youcantfindme"}) assert.Equal(t, expectedWord, actualWord) assert.Equal(t, expectedIndex, actualIndex) // test find one word expectedWord, expectedIndex = "文", 17 actualWord, actualIndex = findFirstMatchedWord(text, []string{"文"}) assert.Equal(t, expectedWord, actualWord) assert.Equal(t, expectedIndex, actualIndex) // test find multiple matched words expectedWord, expectedIndex = "Hello", 0 actualWord, actualIndex = findFirstMatchedWord(text, []string{"Hello", "文"}) assert.Equal(t, expectedWord, actualWord) assert.Equal(t, expectedIndex, actualIndex) } func TestGetRuneRange(t *testing.T) { var ( expectedBegin, expectedEnd, actualBegin, actualEnd int ) runeText := []rune("Hello, I have 中文 and 😂.") runeLen := len(runeText) // test get range of negative offset and negative limit expectedBegin, expectedEnd = 0, 0 actualBegin, actualEnd = getRuneRange(runeText, -1, -1) assert.Equal(t, expectedBegin, actualBegin) assert.Equal(t, expectedEnd, actualEnd) // test get range of exceeding offset and exceeding limit expectedBegin, expectedEnd = runeLen, runeLen actualBegin, actualEnd = getRuneRange(runeText, runeLen+1, runeLen+1) assert.Equal(t, expectedBegin, actualBegin) assert.Equal(t, expectedEnd, actualEnd) // test get range of normal offset and exceeding limit expectedBegin, expectedEnd = 3, runeLen actualBegin, actualEnd = getRuneRange(runeText, 3, runeLen) assert.Equal(t, expectedBegin, actualBegin) assert.Equal(t, expectedEnd, actualEnd) // test get range of normal offset and normal limit expectedBegin, expectedEnd = 3, 10 actualBegin, actualEnd = getRuneRange(runeText, 3, 7) assert.Equal(t, expectedBegin, actualBegin) assert.Equal(t, expectedEnd, actualEnd) } func TestFetchRangedExcerpt(t *testing.T) { var ( expected, actual string ) // test english string expected = "hello..." actual = FetchRangedExcerpt("

hello world

", "...", 0, 5) assert.Equal(t, expected, actual) // test string with offset expected = "...llo你好..." actual = FetchRangedExcerpt("

hello你好world

", "...", 2, 5) assert.Equal(t, expected, actual) // test mixed string with emoticon with offset expected = "...你好😂..." actual = FetchRangedExcerpt("

hello你好😂world

", "...", 5, 3) assert.Equal(t, expected, actual) // test mixed string with offset and exceeding limit expected = "...你好😂world" actual = FetchRangedExcerpt("

hello你好😂world

", "...", 5, 100) assert.Equal(t, expected, actual) } func TestCutLongTitle(t *testing.T) { // Short title, no cutting needed short := "hello" assert.Equal(t, short, cutLongTitle(short)) // Exactly max bytes, no cutting needed exact150 := strings.Repeat("a", 150) assert.Len(t, cutLongTitle(exact150), 150) // Just over max bytes, should be cut exact151 := strings.Repeat("a", 151) assert.Len(t, cutLongTitle(exact151), 150) // Multi-byte rune at boundary gets removed properly asciiPart := strings.Repeat("a", 149) // 149 bytes multiByteChar := "中" // 3 bytes - will span bytes 149-151 title := asciiPart + multiByteChar // 152 bytes total assert.Equal(t, asciiPart, cutLongTitle(title)) } func TestFetchMatchedExcerpt(t *testing.T) { var ( expected, actual string ) html := "

Hello, I have 中文 and 😂 and I am supposed to work fine

" // test find nothing // it should return from the begin with double trimLength text expected = "Hello, I h..." actual = FetchMatchedExcerpt(html, []string{"youcantfindme"}, "...", 5) assert.Equal(t, expected, actual) // test find the word at the end // it should return the word beginning with double trimLenth plus len(word) expected = "... work fine" actual = FetchMatchedExcerpt(html, []string{"youcant", "fine"}, "...", 3) assert.Equal(t, expected, actual) // test find multiple words // it should return the first matched word with trimmedText expected = "... have 中文 and 😂..." actual = FetchMatchedExcerpt(html, []string{"中文", "😂"}, "...", 6) assert.Equal(t, expected, actual) }