chore: import upstream snapshot with attribution

2026-07-13 13:34:58 +08:00
commit a203934033
1368 changed files with 175001 additions and 0 deletions
@@ -0,0 +1,20 @@
+from swift.dataset import load_dataset
+
+
+def test_local_dataset():
+    # please use git clone
+    from swift.utils import git_clone_github
+    model_dir = git_clone_github('https://www.modelscope.cn/datasets/swift/swift-sft-mixture.git')
+    dataset = load_dataset(datasets=[f'{model_dir}:firefly'], streaming=True)[0]
+    print(next(iter(dataset)))
+
+
+def test_hub_dataset():
+    local_dataset = 'swift/swift-sft-mixture:firefly'
+    dataset = load_dataset(datasets=[local_dataset], streaming=True)[0]
+    print(next(iter(dataset)))
+
+
+if __name__ == '__main__':
+    test_local_dataset()
+    # test_hub_dataset()