chore: import upstream snapshot with attribution

tests/test_string_util.py

@@ -0,0 +1,19 @@
+# -*- coding:utf-8 -*-
+# Author: hankcs
+# Date: 2022-03-22 17:17
+import unittest
+
+from hanlp.utils.string_util import possible_tokenization
+
+
+class TestStringUtility(unittest.TestCase):
+    def test_enumerate_tokenization(self):
+        text = '商品和服务'
+        toks = possible_tokenization(text)
+        assert len(set(toks)) == 2 ** (len(text) - 1)
+        for each in toks:
+            assert ''.join(each) == text
+
+
+if __name__ == '__main__':
+    unittest.main()