wehub-resource-sync
264 lines
11 KiB
Python
264 lines
11 KiB
Python
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import copy
|
|
import tempfile
|
|
import unittest
|
|
|
|
import paddle
|
|
from parameterized import parameterized
|
|
|
|
from paddlenlp.prompt import ManualVerbalizer, MaskedLMVerbalizer, SoftVerbalizer
|
|
from paddlenlp.prompt.verbalizer import MaskedLMIdentity
|
|
from paddlenlp.transformers import AutoModelForMaskedLM, AutoTokenizer
|
|
from paddlenlp.transformers.albert.modeling import AlbertMLMHead
|
|
from paddlenlp.transformers.ernie.modeling import ErnieLMPredictionHead
|
|
|
|
|
|
class VerbalizerTest(unittest.TestCase):
|
|
"""
|
|
Unittest for Verbalizer
|
|
"""
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.tokenizer = AutoTokenizer.from_pretrained("ernie-3.0-tiny-nano-v2-zh")
|
|
cls.model = AutoModelForMaskedLM.from_pretrained("ernie-3.0-tiny-nano-v2-zh")
|
|
cls.default_label_words = {"正向": "很", "负向": "不"}
|
|
cls.kwargs = {
|
|
"token_aggregate_type": "first",
|
|
"word_aggregate_type": "first",
|
|
"mask_aggregate_type": "first",
|
|
"post_log_softmax": False,
|
|
}
|
|
cls.default_verb = ManualVerbalizer(label_words=cls.default_label_words, tokenizer=cls.tokenizer)
|
|
|
|
@parameterized.expand(
|
|
[
|
|
(ManualVerbalizer,),
|
|
(SoftVerbalizer,),
|
|
(MaskedLMVerbalizer,),
|
|
]
|
|
)
|
|
def test_kwargs(self, class_name):
|
|
model = copy.deepcopy(self.model)
|
|
verb = class_name(tokenizer=self.tokenizer, label_words=self.default_label_words, model=model, **self.kwargs)
|
|
self.assertEqual(verb.token_aggregate_type, "first")
|
|
self.assertEqual(verb.word_aggregate_type, "first")
|
|
self.assertEqual(verb.mask_aggregate_type, "first")
|
|
self.assertFalse(verb.post_log_softmax)
|
|
|
|
def test_labels_property(self):
|
|
verb = ManualVerbalizer(label_words=self.default_label_words, tokenizer=self.tokenizer)
|
|
labels = ["差评", "好评"]
|
|
label_words = {"差评": "避雷", "好评": "非常推荐"}
|
|
expected_words = {"好评": ["非常推荐"], "差评": ["避雷"]}
|
|
expected_token_ids = paddle.to_tensor([[[465, 223, 426, 1645]], [[1166, 1048, 0, 0]]], dtype="int64")
|
|
expected_word_mask = paddle.to_tensor([[1], [1]], dtype="int64")
|
|
expected_token_mask = paddle.to_tensor([[[1, 1, 1, 1]], [[1, 1, 0, 0]]], dtype="int64")
|
|
with self.assertRaises(NotImplementedError):
|
|
verb.labels = labels
|
|
verb.label_words = label_words
|
|
self.assertEqual(verb.labels, sorted(labels))
|
|
self.assertEqual(verb.label_words, expected_words)
|
|
self.assertTrue(paddle.equal_all(verb.token_ids, expected_token_ids))
|
|
self.assertTrue(paddle.equal_all(verb.word_mask, expected_word_mask))
|
|
self.assertTrue(paddle.equal_all(verb.token_mask, expected_token_mask))
|
|
|
|
@parameterized.expand(
|
|
[
|
|
("mean", [[0.5, 2.0], [-3.0, -1.5]]),
|
|
("max", [[1.0, 2.0], [-3.0, -1.0]]),
|
|
("first", [[0.0, 2.0], [-3.0, -1.0]]),
|
|
]
|
|
)
|
|
def test_aggregate_token(self, atype, expected_outputs):
|
|
outputs = paddle.to_tensor([[[0, 1.0], [2, 3.0]], [[-3, -4.0], [-1, -2.0]]])
|
|
token_mask = paddle.to_tensor([[[1, 1], [1, 0]], [[1, 0], [1, 1]]])
|
|
outputs = self.default_verb.aggregate(outputs, token_mask, atype)
|
|
self.assertEqual(outputs.tolist(), expected_outputs)
|
|
|
|
@parameterized.expand(
|
|
[
|
|
("mean", [0.5, 2.0]),
|
|
("max", [1.0, 2.0]),
|
|
("first", [0.0, 2.0]),
|
|
]
|
|
)
|
|
def test_aggregate_word(self, atype, expected_outputs):
|
|
outputs = paddle.to_tensor([[0, 1.0], [2, 3.0]])
|
|
word_mask = paddle.to_tensor([[1, 1], [1, 0]])
|
|
outputs = self.default_verb.aggregate(outputs, word_mask, atype)
|
|
self.assertEqual(outputs.tolist(), expected_outputs)
|
|
|
|
def test_project(self):
|
|
outputs = self.default_verb.project(paddle.rand([2, 1, 400]))
|
|
self.assertEqual(outputs.shape, [2, 1, 2, 1])
|
|
|
|
def test_normalize(self):
|
|
outputs = paddle.rand([2, 1, 2, 3])
|
|
self.assertAlmostEqual(self.default_verb.normalize(outputs)[0].sum().item(), 1, 6)
|
|
self.assertAlmostEqual(self.default_verb.normalize(outputs)[1].sum().item(), 1, 6)
|
|
|
|
def test_save_and_load(self):
|
|
with tempfile.TemporaryDirectory() as tmpdirname:
|
|
self.default_verb.save(save_path=tmpdirname)
|
|
verb = ManualVerbalizer.load_from(tmpdirname, tokenizer=self.tokenizer)
|
|
self.assertEqual(verb.label_words, self.default_verb.label_words)
|
|
|
|
def test_encode_and_decode(self):
|
|
label_words = {"负向": "不喜欢", "正向": "非常推荐"}
|
|
verb = ManualVerbalizer(label_words=label_words, tokenizer=self.tokenizer)
|
|
self.assertEqual(verb.convert_ids_to_labels(1), "负向")
|
|
self.assertEqual(verb.convert_labels_to_ids("负向"), 1)
|
|
|
|
@parameterized.expand(
|
|
[
|
|
(
|
|
{"0": "负向", "1": "正向"},
|
|
["0", "1"],
|
|
{"0": ["负向"], "1": ["正向"]},
|
|
[[[383, 253]], [[243, 253]]],
|
|
[[1], [1]],
|
|
[[[1, 1]], [[1, 1]]],
|
|
),
|
|
(
|
|
{0: ["差评", "不喜欢"], 1: ["好评", "不错"]},
|
|
[0, 1],
|
|
{0: ["差评", "不喜欢"], 1: ["好评", "不错"]},
|
|
[[[859, 480, 0], [16, 692, 811]], [[170, 480, 0], [16, 990, 0]]],
|
|
[[1, 1], [1, 1]],
|
|
[[[1, 1, 0], [1, 1, 1]], [[1, 1, 0], [1, 1, 0]]],
|
|
),
|
|
(
|
|
{1: ["很满意", "非常推荐"], 0: "避雷"},
|
|
[0, 1],
|
|
{0: ["避雷"], 1: ["很满意", "非常推荐"]},
|
|
[[[1166, 1048, 0, 0], [0, 0, 0, 0]], [[321, 596, 221, 0], [465, 223, 426, 1645]]],
|
|
[[1, 0], [1, 1]],
|
|
[[[1, 1, 0, 0], [0, 0, 0, 0]], [[1, 1, 1, 0], [1, 1, 1, 1]]],
|
|
),
|
|
]
|
|
)
|
|
def test_manual_initialization(
|
|
self, label_words, labels, expected_words, expected_token_ids, expected_word_mask, expected_token_mask
|
|
):
|
|
verb = ManualVerbalizer(label_words=label_words, tokenizer=self.tokenizer)
|
|
self.assertEqual(verb.labels, labels)
|
|
self.assertEqual(verb.label_words, expected_words)
|
|
self.assertTrue(paddle.equal_all(verb.token_ids, paddle.to_tensor(expected_token_ids, dtype="int64")))
|
|
self.assertTrue(paddle.equal_all(verb.word_mask, paddle.to_tensor(expected_word_mask, dtype="int64")))
|
|
self.assertTrue(paddle.equal_all(verb.token_mask, paddle.to_tensor(expected_token_mask, dtype="int64")))
|
|
|
|
@parameterized.expand(
|
|
[
|
|
("mean", [[1, 2.0], [-2, -3.0]]),
|
|
("max", [[2, 3.0], [-1, -2.0]]),
|
|
("first", [[0, 1.0], [-3, -4.0]]),
|
|
("product", [[0, 3.0], [3, 8.0]]),
|
|
("invalid", None),
|
|
]
|
|
)
|
|
def test_manual_aggregate_multiple_mask(self, atype, expected_outputs):
|
|
outputs = paddle.to_tensor([[[0, 1.0], [2, 3.0]], [[-3, -4.0], [-1, -2.0]]])
|
|
if atype == "invalid":
|
|
with self.assertRaises(ValueError):
|
|
self.default_verb.aggregate_multiple_mask(outputs, atype)
|
|
else:
|
|
outputs = self.default_verb.aggregate_multiple_mask(outputs, atype)
|
|
self.assertEqual(outputs.tolist(), expected_outputs)
|
|
|
|
def test_manual_process_outputs(self):
|
|
outputs = paddle.rand([3, 2, 500])
|
|
masked_positions = paddle.to_tensor([0, 2, 5])
|
|
outputs = self.default_verb.process_outputs(outputs, masked_positions)
|
|
self.assertEqual(outputs.shape, [3, 2])
|
|
|
|
@parameterized.expand(
|
|
[
|
|
(
|
|
"__internal_testing__/tiny-random-ernie",
|
|
["cls", "predictions", "decoder"],
|
|
ErnieLMPredictionHead,
|
|
["decoder_bias", "decoder.weight", "decoder.bias"],
|
|
["transform.weight", "transform.bias", "layer_norm.weight", "layer_norm.bias"],
|
|
),
|
|
(
|
|
"albert-chinese-tiny",
|
|
["predictions", "decoder"],
|
|
AlbertMLMHead,
|
|
["bias", "decoder.weight", "decoder.bias"],
|
|
["layer_norm.weight", "layer_norm.bias", "dense.weight", "dense.bias"],
|
|
),
|
|
]
|
|
)
|
|
def test_soft_initialization(self, model_name, head_name, head_class, head_params, non_head_params):
|
|
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
|
model = AutoModelForMaskedLM.from_pretrained(model_name, ignore_mismatched_sizes=True)
|
|
verb = SoftVerbalizer(self.default_label_words, tokenizer, model)
|
|
self.assertEqual(verb.head_name, head_name)
|
|
self.assertTrue(isinstance(verb.head, head_class))
|
|
self.assertTrue(isinstance(getattr(model, head_name[0]), MaskedLMIdentity))
|
|
module = getattr(verb.head, verb.head_name[-1])
|
|
# module = module.weight if isinstance(module, paddle.nn.Linear) else module
|
|
self.assertTrue(len(self.default_label_words) in module.weight.shape)
|
|
self.assertEqual([x[0] for x in verb.head_parameters()], head_params)
|
|
self.assertEqual([x[0] for x in verb.non_head_parameters()], non_head_params)
|
|
|
|
def test_soft_process_outputs(self):
|
|
verb = SoftVerbalizer(self.default_label_words, self.tokenizer, copy.deepcopy(self.model))
|
|
outputs = paddle.rand([3, 2, 312])
|
|
masked_positions = paddle.to_tensor([0, 2, 5])
|
|
outputs = verb.process_outputs(outputs, masked_positions)
|
|
|
|
@parameterized.expand(
|
|
[
|
|
(
|
|
{"0": "负向", "1": "正向"},
|
|
{"0": ["负向"], "1": ["正向"]},
|
|
),
|
|
(
|
|
{1: ["好评", "不错"], 0: ["差评", "不喜欢"]},
|
|
{0: ["差评"], 1: ["好评"]},
|
|
),
|
|
(
|
|
{1: "很满意", 0: "避雷"},
|
|
None,
|
|
),
|
|
]
|
|
)
|
|
def test_maskedlm_initialization(self, label_words, expected_label_words):
|
|
if expected_label_words is None:
|
|
with self.assertRaises(ValueError):
|
|
verb = MaskedLMVerbalizer(label_words, self.tokenizer)
|
|
else:
|
|
verb = MaskedLMVerbalizer(label_words, self.tokenizer)
|
|
self.assertEqual(verb.label_words, expected_label_words)
|
|
|
|
@parameterized.expand([("mean",), ("max",), ("first",), ("product",), ("sum",), ("invalid",)])
|
|
def test_maskedlm_aggregate_multiple_mask(self, atype):
|
|
label_words = {"0": "负向", "1": "正向"}
|
|
verb = MaskedLMVerbalizer(label_words, self.tokenizer)
|
|
outputs = paddle.rand([3, 2, 500])
|
|
if atype == "invalid":
|
|
with self.assertRaises(ValueError):
|
|
verb.aggregate_multiple_mask(outputs, atype)
|
|
else:
|
|
outputs = verb.aggregate_multiple_mask(outputs, atype)
|
|
self.assertEqual(outputs.shape, [3, 2])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|