chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,319 @@
|
||||
package encoding
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf16"
|
||||
|
||||
"golang.org/x/text/encoding/simplifiedchinese"
|
||||
)
|
||||
|
||||
// --- Detect ---
|
||||
|
||||
func TestDetectUTF8Plain(t *testing.T) {
|
||||
enc, _ := Detect([]byte("hello world\n"))
|
||||
if enc != UTF8 {
|
||||
t.Errorf("got %v, want UTF8", enc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectUTF8BOM(t *testing.T) {
|
||||
in := append([]byte{0xEF, 0xBB, 0xBF}, []byte("hello")...)
|
||||
enc, _ := Detect(in)
|
||||
if enc != UTF8BOM {
|
||||
t.Errorf("got %v, want UTF8BOM", enc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectUTF16LE(t *testing.T) {
|
||||
var b bytes.Buffer
|
||||
b.Write([]byte{0xFF, 0xFE})
|
||||
for _, r := range utf16.Encode([]rune("hello")) {
|
||||
_ = binary.Write(&b, binary.LittleEndian, r)
|
||||
}
|
||||
enc, _ := Detect(b.Bytes())
|
||||
if enc != UTF16LE {
|
||||
t.Errorf("got %v, want UTF16LE", enc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectUTF16BE(t *testing.T) {
|
||||
var b bytes.Buffer
|
||||
b.Write([]byte{0xFE, 0xFF})
|
||||
for _, r := range utf16.Encode([]rune("hello")) {
|
||||
_ = binary.Write(&b, binary.BigEndian, r)
|
||||
}
|
||||
enc, _ := Detect(b.Bytes())
|
||||
if enc != UTF16BE {
|
||||
t.Errorf("got %v, want UTF16BE", enc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectGB18030(t *testing.T) {
|
||||
gb, err := simplifiedchinese.GB18030.NewEncoder().String("你好世界")
|
||||
if err != nil {
|
||||
t.Fatalf("encode: %v", err)
|
||||
}
|
||||
enc, _ := Detect([]byte(gb))
|
||||
if enc != GB18030 {
|
||||
t.Errorf("got %v, want GB18030", enc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectEmpty(t *testing.T) {
|
||||
enc, _ := Detect(nil)
|
||||
if enc != UTF8 {
|
||||
t.Errorf("empty input: got %v, want UTF8", enc)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Decode ---
|
||||
|
||||
func TestDecodeUTF8(t *testing.T) {
|
||||
in := []byte("hello\n世界")
|
||||
out := Decode(in, UTF8)
|
||||
if string(out) != "hello\n世界" {
|
||||
t.Errorf("got %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadFileUTF8DecodesGB18030(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "settings.json")
|
||||
if err := os.WriteFile(path, Encode(`{"label":"中文"}`, GB18030), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := ReadFileUTF8(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got) != `{"label":"中文"}` {
|
||||
t.Fatalf("ReadFileUTF8 = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeUTF8BOM(t *testing.T) {
|
||||
in := append([]byte{0xEF, 0xBB, 0xBF}, []byte("hello")...)
|
||||
out := Decode(in, UTF8BOM)
|
||||
if string(out) != "hello" {
|
||||
t.Errorf("got %q, want 'hello'", out)
|
||||
}
|
||||
if bytes.Contains(out, []byte{0xEF, 0xBB, 0xBF}) {
|
||||
t.Error("BOM leaked into decoded output")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeUTF16LE(t *testing.T) {
|
||||
var b bytes.Buffer
|
||||
b.Write([]byte{0xFF, 0xFE})
|
||||
for _, r := range utf16.Encode([]rune("hello\nworld")) {
|
||||
_ = binary.Write(&b, binary.LittleEndian, r)
|
||||
}
|
||||
out := Decode(b.Bytes(), UTF16LE)
|
||||
if string(out) != "hello\nworld" {
|
||||
t.Errorf("got %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeUTF16BE(t *testing.T) {
|
||||
var b bytes.Buffer
|
||||
b.Write([]byte{0xFE, 0xFF})
|
||||
for _, r := range utf16.Encode([]rune("hello\nworld")) {
|
||||
_ = binary.Write(&b, binary.BigEndian, r)
|
||||
}
|
||||
out := Decode(b.Bytes(), UTF16BE)
|
||||
if string(out) != "hello\nworld" {
|
||||
t.Errorf("got %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeGB18030(t *testing.T) {
|
||||
gb, _ := simplifiedchinese.GB18030.NewEncoder().String("你好世界\n第二行")
|
||||
out := Decode([]byte(gb), GB18030)
|
||||
if string(out) != "你好世界\n第二行" {
|
||||
t.Errorf("got %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeLossyUTF8(t *testing.T) {
|
||||
in := []byte{0xFF, 0xFE, 'a'}
|
||||
out := Decode(in, LossyUTF8)
|
||||
if !bytes.Equal(out, in) {
|
||||
t.Errorf("LossyUTF8 should pass through, got %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Encode ---
|
||||
|
||||
func TestEncodeUTF8(t *testing.T) {
|
||||
out := Encode("hello", UTF8)
|
||||
if string(out) != "hello" {
|
||||
t.Errorf("got %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeUTF8BOM(t *testing.T) {
|
||||
out := Encode("hello", UTF8BOM)
|
||||
if !bytes.HasPrefix(out, []byte{0xEF, 0xBB, 0xBF}) {
|
||||
t.Error("missing UTF-8 BOM prefix")
|
||||
}
|
||||
if string(out[3:]) != "hello" {
|
||||
t.Errorf("body = %q", out[3:])
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeUTF16LE(t *testing.T) {
|
||||
out := Encode("hi", UTF16LE)
|
||||
if len(out) < 2 || out[0] != 0xFF || out[1] != 0xFE {
|
||||
t.Error("missing UTF-16LE BOM")
|
||||
}
|
||||
decoded := Decode(out, UTF16LE)
|
||||
if string(decoded) != "hi" {
|
||||
t.Errorf("round-trip failed: got %q", decoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeUTF16BE(t *testing.T) {
|
||||
out := Encode("hi", UTF16BE)
|
||||
if len(out) < 2 || out[0] != 0xFE || out[1] != 0xFF {
|
||||
t.Error("missing UTF-16BE BOM")
|
||||
}
|
||||
decoded := Decode(out, UTF16BE)
|
||||
if string(decoded) != "hi" {
|
||||
t.Errorf("round-trip failed: got %q", decoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeGB18030(t *testing.T) {
|
||||
out := Encode("你好", GB18030)
|
||||
dec, _ := simplifiedchinese.GB18030.NewDecoder().Bytes(out)
|
||||
if string(dec) != "你好" {
|
||||
t.Errorf("round-trip failed: got %q", dec)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Round-trip ---
|
||||
|
||||
func TestRoundTripGB18030(t *testing.T) {
|
||||
original := "你好世界\n第二行\n"
|
||||
gb, _ := simplifiedchinese.GB18030.NewEncoder().String(original)
|
||||
|
||||
enc, _ := Detect([]byte(gb))
|
||||
decoded := string(Decode([]byte(gb), enc))
|
||||
if decoded != original {
|
||||
t.Fatalf("decode mismatch: %q", decoded)
|
||||
}
|
||||
|
||||
edited := strings.Replace(decoded, "第二行", "新的行", 1)
|
||||
reencoded := Encode(edited, enc)
|
||||
redecoded := string(Decode(reencoded, enc))
|
||||
if redecoded != edited {
|
||||
t.Errorf("round-trip failed: got %q, want %q", redecoded, edited)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoundTripUTF16LE(t *testing.T) {
|
||||
original := "hello\nworld\n"
|
||||
encoded := Encode(original, UTF16LE)
|
||||
enc, _ := Detect(encoded)
|
||||
decoded := string(Decode(encoded, enc))
|
||||
if decoded != original {
|
||||
t.Errorf("round-trip failed: got %q, want %q", decoded, original)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoundTripUTF8BOM(t *testing.T) {
|
||||
original := "hello world\n"
|
||||
encoded := Encode(original, UTF8BOM)
|
||||
enc, _ := Detect(encoded)
|
||||
decoded := string(Decode(encoded, enc))
|
||||
if decoded != original {
|
||||
t.Errorf("round-trip failed: got %q, want %q", decoded, original)
|
||||
}
|
||||
}
|
||||
|
||||
// --- BOM-less UTF-16 ---
|
||||
|
||||
func utf16NoBOM(t *testing.T, s string, order binary.ByteOrder) []byte {
|
||||
t.Helper()
|
||||
var b bytes.Buffer
|
||||
for _, r := range utf16.Encode([]rune(s)) {
|
||||
_ = binary.Write(&b, order, r)
|
||||
}
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func TestDetectUTF16LENoBOM(t *testing.T) {
|
||||
in := utf16NoBOM(t, "// Created by 69431 on 2024/12/31\n#include \"x.h\"\n", binary.LittleEndian)
|
||||
enc, _ := Detect(in)
|
||||
if enc != UTF16LENoBOM {
|
||||
t.Errorf("got %v, want UTF16LENoBOM", enc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectUTF16BENoBOM(t *testing.T) {
|
||||
in := utf16NoBOM(t, "package main\nfunc main() {}\n", binary.BigEndian)
|
||||
enc, _ := Detect(in)
|
||||
if enc != UTF16BENoBOM {
|
||||
t.Errorf("got %v, want UTF16BENoBOM", enc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectPlainUTF8NotUTF16(t *testing.T) {
|
||||
enc, _ := Detect([]byte("the quick brown fox jumps over the lazy dog\n"))
|
||||
if enc != UTF8 {
|
||||
t.Errorf("ASCII text misdetected as %v", enc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectBinaryNotUTF16NoBOM(t *testing.T) {
|
||||
// NULs on both parities — genuine binary must not look like BOM-less UTF-16.
|
||||
bin := []byte{0x00, 0x00, 0x01, 0x00, 0x00, 0x02, 0xFF, 0x00, 0x00, 0x03, 0x00, 0x00, 0x04, 0x00, 0x00, 0x05, 0x00, 0x00}
|
||||
if k, ok := DetectUTF16NoBOM(bin); ok {
|
||||
t.Errorf("binary misdetected as %v", k)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeUTF16LENoBOM(t *testing.T) {
|
||||
in := utf16NoBOM(t, "hello\nworld", binary.LittleEndian)
|
||||
if out := string(Decode(in, UTF16LENoBOM)); out != "hello\nworld" {
|
||||
t.Errorf("got %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoundTripUTF16LENoBOM(t *testing.T) {
|
||||
original := "// c++ source\nint main() { return 0; }\n"
|
||||
encoded := utf16NoBOM(t, original, binary.LittleEndian)
|
||||
|
||||
enc, _ := Detect(encoded)
|
||||
if enc != UTF16LENoBOM {
|
||||
t.Fatalf("detect: got %v", enc)
|
||||
}
|
||||
decoded := string(Decode(encoded, enc))
|
||||
if decoded != original {
|
||||
t.Fatalf("decode mismatch: %q", decoded)
|
||||
}
|
||||
edited := strings.Replace(decoded, "return 0", "return 1", 1)
|
||||
reencoded := Encode(edited, enc)
|
||||
if bytes.HasPrefix(reencoded, []byte{0xFF, 0xFE}) || bytes.HasPrefix(reencoded, []byte{0xFE, 0xFF}) {
|
||||
t.Error("no-BOM re-encode leaked a BOM")
|
||||
}
|
||||
if redecoded := string(Decode(reencoded, enc)); redecoded != edited {
|
||||
t.Errorf("round-trip failed: got %q, want %q", redecoded, edited)
|
||||
}
|
||||
}
|
||||
|
||||
// --- UTF-16 supplementary plane (surrogate pairs) ---
|
||||
|
||||
func TestSurrogatePairRoundTrip(t *testing.T) {
|
||||
// U+1F600 (😀) is in the supplementary plane and requires a surrogate pair.
|
||||
original := "hello 😀 world"
|
||||
encoded := Encode(original, UTF16LE)
|
||||
decoded := string(Decode(encoded, UTF16LE))
|
||||
if decoded != original {
|
||||
t.Errorf("surrogate pair round-trip failed: got %q, want %q", decoded, original)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user