wehub-resource-sync
144 lines
5.3 KiB
Python
144 lines
5.3 KiB
Python
# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""
|
|
文本处理工具测试 / Text Processing Utility Tests
|
|
|
|
测试目标 / Test Target:
|
|
paddle.text 文本处理功能
|
|
|
|
覆盖的模块 / Covered Modules:
|
|
- paddle.nn.Embedding: 词嵌入
|
|
- paddle.nn.functional.one_hot: One-hot编码
|
|
- paddle.nn.functional.label_smooth: 标签平滑
|
|
- paddle.text: 文本数据集
|
|
|
|
作用 / Purpose:
|
|
补充文本处理API的测试，提升覆盖率。
|
|
"""
|
|
|
|
import unittest
|
|
|
|
import numpy as np
|
|
|
|
import paddle
|
|
import paddle.nn.functional as F
|
|
from paddle import nn
|
|
|
|
paddle.disable_static()
|
|
|
|
|
|
class TestOneHot(unittest.TestCase):
|
|
"""测试One-Hot编码 / Test one-hot encoding"""
|
|
|
|
def test_one_hot_basic(self):
|
|
"""测试基本One-Hot / Test basic one-hot"""
|
|
x = paddle.to_tensor([0, 1, 2, 3])
|
|
result = F.one_hot(x, num_classes=5)
|
|
self.assertEqual(result.shape, [4, 5])
|
|
# Check correctness
|
|
np.testing.assert_allclose(result[0].numpy(), [1, 0, 0, 0, 0])
|
|
np.testing.assert_allclose(result[1].numpy(), [0, 1, 0, 0, 0])
|
|
|
|
def test_one_hot_2d(self):
|
|
"""测试2D输入One-Hot / Test one-hot with 2D input"""
|
|
x = paddle.to_tensor([[0, 1], [2, 3]])
|
|
result = F.one_hot(x, num_classes=5)
|
|
self.assertEqual(result.shape, [2, 2, 5])
|
|
|
|
def test_one_hot_last_class(self):
|
|
"""测试最后一类One-Hot / Test one-hot for last class"""
|
|
x = paddle.to_tensor([4])
|
|
result = F.one_hot(x, num_classes=5)
|
|
np.testing.assert_allclose(result[0].numpy(), [0, 0, 0, 0, 1])
|
|
|
|
|
|
class TestLabelSmooth(unittest.TestCase):
|
|
"""测试标签平滑 / Test label smoothing"""
|
|
|
|
def test_label_smooth_basic(self):
|
|
"""测试基本标签平滑 / Test basic label smoothing"""
|
|
label = paddle.to_tensor([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])
|
|
smoothed = F.label_smooth(label, epsilon=0.1)
|
|
self.assertEqual(smoothed.shape, [2, 3])
|
|
# After smoothing, sum should still be 1 per row
|
|
row_sums = smoothed.sum(axis=1)
|
|
np.testing.assert_allclose(row_sums.numpy(), [1.0, 1.0], rtol=1e-5)
|
|
|
|
def test_label_smooth_epsilon(self):
|
|
"""测试不同epsilon的标签平滑 / Test label smooth with different epsilon"""
|
|
label = paddle.to_tensor([[1.0, 0.0]])
|
|
smoothed = F.label_smooth(label, epsilon=0.2)
|
|
# Maximum value should be less than 1.0
|
|
self.assertLess(float(smoothed.max().numpy()), 1.0)
|
|
|
|
|
|
class TestSequenceOps(unittest.TestCase):
|
|
"""测试序列操作 / Test sequence operations"""
|
|
|
|
def test_embedding_bag(self):
|
|
"""测试EmbeddingBag模拟 / Test EmbeddingBag simulation"""
|
|
emb = nn.Embedding(num_embeddings=100, embedding_dim=16)
|
|
x = paddle.to_tensor([0, 1, 2, 3, 4, 5])
|
|
# Simulate bag aggregation: two bags [0,1,2] and [3,4,5]
|
|
bag1 = emb(x[:3]).mean(axis=0)
|
|
bag2 = emb(x[3:]).mean(axis=0)
|
|
result = paddle.stack([bag1, bag2])
|
|
self.assertEqual(result.shape, [2, 16])
|
|
|
|
def test_embedding_sequential(self):
|
|
"""测试序列嵌入 / Test sequential embedding"""
|
|
# Simulate sequence processing
|
|
vocab_size = 50
|
|
d_model = 16
|
|
emb = nn.Embedding(vocab_size, d_model)
|
|
# Batch of 4 sequences, each length 10
|
|
tokens = paddle.randint(0, vocab_size, [4, 10])
|
|
embedded = emb(tokens)
|
|
self.assertEqual(embedded.shape, [4, 10, d_model])
|
|
|
|
|
|
class TestCrossEntropyVariants(unittest.TestCase):
|
|
"""测试交叉熵变体 / Test cross-entropy variants"""
|
|
|
|
def test_cross_entropy_hard_label(self):
|
|
"""测试硬标签交叉熵 / Test hard label cross entropy"""
|
|
logits = paddle.randn([4, 10])
|
|
labels = paddle.randint(0, 10, [4])
|
|
loss = F.cross_entropy(logits, labels)
|
|
self.assertGreater(float(loss.numpy()), 0)
|
|
|
|
def test_cross_entropy_soft_label(self):
|
|
"""测试软标签交叉熵 / Test soft label cross entropy"""
|
|
logits = paddle.randn([4, 10])
|
|
labels = paddle.ones([4, 10]) / 10 # Uniform soft labels
|
|
loss = F.cross_entropy(logits, labels, soft_label=True)
|
|
self.assertGreater(float(loss.numpy()), 0)
|
|
|
|
def test_cross_entropy_reduction(self):
|
|
"""测试不同归约方式的交叉熵 / Test CE with different reductions"""
|
|
logits = paddle.randn([4, 10])
|
|
labels = paddle.randint(0, 10, [4])
|
|
loss_mean = F.cross_entropy(logits, labels, reduction='mean')
|
|
loss_sum = F.cross_entropy(logits, labels, reduction='sum')
|
|
loss_none = F.cross_entropy(logits, labels, reduction='none')
|
|
self.assertEqual(loss_none.shape, [4])
|
|
self.assertAlmostEqual(
|
|
float(loss_sum.numpy()), float(loss_none.sum().numpy()), places=4
|
|
)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|