chore: import upstream snapshot with attribution

2026-07-13 13:34:58 +08:00
commit a203934033
1368 changed files with 175001 additions and 0 deletions
@@ -0,0 +1,30 @@
+import os
+from transformers import FeatureExtractionMixin, PreTrainedTokenizerBase
+from transformers import ProcessorMixin as HfProcessorMixin
+from typing import Union
+
+try:
+    from transformers import BaseImageProcessor
+    Processor = Union[PreTrainedTokenizerBase, BaseImageProcessor, FeatureExtractionMixin, HfProcessorMixin]
+except ImportError:
+    Processor = Union[PreTrainedTokenizerBase, FeatureExtractionMixin, HfProcessorMixin]
+
+if 'TOKENIZERS_PARALLELISM' not in os.environ:
+    os.environ['TOKENIZERS_PARALLELISM'] = 'false'
+
+
+class ProcessorMixin:
+
+    @property
+    def tokenizer(self):
+        tokenizer = self.processor
+        if not isinstance(tokenizer, PreTrainedTokenizerBase) and hasattr(tokenizer, 'tokenizer'):
+            tokenizer = tokenizer.tokenizer
+        return tokenizer
+
+    @tokenizer.setter
+    def tokenizer(self, value):
+        if self.processor is self.tokenizer:
+            self.processor = value
+        elif self.tokenizer is not value:
+            raise AttributeError('Please use `self.processor` for assignment.')