wehub-resource-sync
30 lines
750 B
Python
30 lines
750 B
Python
import pytest
|
|
|
|
from ray.data.preprocessors.utils import simple_hash, simple_split_tokenizer
|
|
|
|
|
|
def test_simple_split_tokenizer():
|
|
# Tests simple_split_tokenizer.
|
|
assert simple_split_tokenizer("one_word") == ["one_word"]
|
|
assert simple_split_tokenizer("two words") == ["two", "words"]
|
|
assert simple_split_tokenizer("One fish. Two fish.") == [
|
|
"One",
|
|
"fish.",
|
|
"Two",
|
|
"fish.",
|
|
]
|
|
|
|
|
|
def test_simple_hash():
|
|
# Tests simple_hash determinism.
|
|
assert simple_hash(1, 100) == 15
|
|
assert simple_hash("a", 100) == 99
|
|
assert simple_hash("banana", 100) == 10
|
|
assert simple_hash([1, 2, "apple"], 100) == 58
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import sys
|
|
|
|
sys.exit(pytest.main(["-sv", __file__]))
|