# Copyright 2022 HuggingFace Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest from transformers import DonutProcessor from ...test_processing_common import ProcessorTesterMixin class DonutProcessorTest(ProcessorTesterMixin, unittest.TestCase): # Tiny processor created with make_tiny_processor.py from "naver-clova-ix/donut-base" tiny_model_id = "hf-internal-testing/tiny-processor-donut" processor_class = DonutProcessor @classmethod def _setup_image_processor(cls): image_processor_class = cls._get_component_class_from_processor("image_processor") # Default size=2560×1920 is the document-scanning resolution (~59 MB per image as float32). # Use 64×64 for tests — no assertions check spatial dimensions. return image_processor_class.from_pretrained(cls.tiny_model_id, size={"height": 64, "width": 64}) def test_token2json(self): expected_json = { "name": "John Doe", "age": "99", "city": "Atlanta", "state": "GA", "zip": "30301", "phone": "123-4567", "nicknames": [{"nickname": "Johnny"}, {"nickname": "JD"}], "multiline": "text\nwith\nnewlines", "empty": "", } sequence = ( "John Doe99Atlanta" "GA30301123-4567" "Johnny" "JD" "text\nwith\nnewlines" "" ) processor = self.get_processor() actual_json = processor.token2json(sequence) self.assertDictEqual(actual_json, expected_json)