'''simple docstring'''






import unittest

from transformers import MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING, AutoTokenizer, is_vision_available
from transformers.pipelines import pipeline
from transformers.pipelines.document_question_answering import apply_tesseract
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_detectrona,
    require_pytesseract,
    require_tf,
    require_torch,
    require_vision,
    slow,
)

from .test_pipelines_common import ANY


if is_vision_available():
     from PIL import Image

     from transformers.image_utils import load_image
else:





     class     A__							:





        """simple docstring"""





        @staticmethod
        def 			_lowerCAmelCase		( *lowerCAmelCase__     :		List[str]						,     **lowerCAmelCase__     :		List[str]			)       ->    Optional[int]:







           """simple docstring"""




           pass






     def    __UpperCAmelCase			(  a_:       int							):
        return None


# This is a pinned image from a specific revision of a document question answering space, hosted by HuggingFace,
# so we can expect it to be available.
__a										=					(
    'https://huggingface.co/spaces/impira/docquery/resolve/2f6c96314dc84dfda62d40de9da55f2f5165d403/invoice.png'
)





@is_pipeline_test
@require_torch
@require_vision
class     A__							(       unittest.TestCase   ):





   """simple docstring"""





   UpperCamelCase_       :						Optional[Any]    				=     MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING




   @require_pytesseract
   @require_vision
   def 			_lowerCAmelCase		( self     :		List[Any]						,     lowerCAmelCase__     :		Tuple						,     lowerCAmelCase__     :		Union[str, Any]						,     lowerCAmelCase__     :		Any			)       ->    Tuple:







      """simple docstring"""




      _UpperCAmelCase							:	int							=   pipeline(
          "document-question-answering"						,     model=a__						,     tokenizer=a__						,     image_processor=a__			)

      _UpperCAmelCase							:	int							=   INVOICE_URL
      _UpperCAmelCase							:	Optional[int]							=   list(zip(*apply_tesseract(load_image(a__			)						,     a__						,     ""			)			)			)
      _UpperCAmelCase							:	Union[str, Any]							=   "What is the placebo?"
      _UpperCAmelCase							:	Union[str, Any]							=   [
          {
              "image": load_image(a__			),
              "question": question,
          },
          {
              "image": image,
              "question": question,
          },
          {
              "image": image,
              "question": question,
              "word_boxes": word_boxes,
          },
      ]
      return dqa_pipeline, examples




   def 			_lowerCAmelCase		( self     :		Union[str, Any]						,     lowerCAmelCase__     :		Optional[int]						,     lowerCAmelCase__     :		Dict			)       ->    Dict:







      """simple docstring"""




      _UpperCAmelCase							:	Dict							=   dqa_pipeline(a__						,     top_k=2			)
      self.assertEqual(
          a__						,     [
              [
                  {"score": ANY(a__			), "answer": ANY(a__			), "start": ANY(a__			), "end": ANY(a__			)},
                  {"score": ANY(a__			), "answer": ANY(a__			), "start": ANY(a__			), "end": ANY(a__			)},
              ]
          ]
          * 3						,     )




   @require_torch
   @require_detectrona
   @require_pytesseract
   def 			_lowerCAmelCase		( self     :		Dict			)       ->    List[Any]:







      """simple docstring"""




      _UpperCAmelCase							:	Union[str, Any]							=   pipeline("document-question-answering"						,     model="hf-internal-testing/tiny-random-layoutlmv2"			)
      _UpperCAmelCase							:	str							=   INVOICE_URL
      _UpperCAmelCase							:	List[str]							=   "How many cats are there?"

      _UpperCAmelCase							:	Union[str, Any]							=   [
          {"score": 0.0001, "answer": "oy 2312/2019", "start": 3_8, "end": 3_9},
          {"score": 0.0001, "answer": "oy 2312/2019 DUE", "start": 3_8, "end": 4_0},
      ]
      _UpperCAmelCase							:	List[str]							=   dqa_pipeline(image=a__						,     question=a__						,     top_k=2			)
      self.assertEqual(nested_simplify(a__						,     decimals=4			)						,     a__			)

      _UpperCAmelCase							:	List[Any]							=   dqa_pipeline({"image": image, "question": question}						,     top_k=2			)
      self.assertEqual(nested_simplify(a__						,     decimals=4			)						,     a__			)

      # This image does not detect ANY text in it, meaning layoutlmv2 should fail.
      # Empty answer probably
      _UpperCAmelCase							:	Optional[Any]							=   "./tests/fixtures/tests_samples/COCO/000000039769.png"
      _UpperCAmelCase							:	Dict							=   dqa_pipeline(image=a__						,     question=a__						,     top_k=2			)
      self.assertEqual(a__						,     []			)

      # We can optionnally pass directly the words and bounding boxes
      _UpperCAmelCase							:	Dict							=   "./tests/fixtures/tests_samples/COCO/000000039769.png"
      _UpperCAmelCase							:	str							=   []
      _UpperCAmelCase							:	str							=   []
      _UpperCAmelCase							:	Union[str, Any]							=   dqa_pipeline(image=a__						,     question=a__						,     words=a__						,     boxes=a__						,     top_k=2			)
      self.assertEqual(a__						,     []			)




   @slow
   @require_torch
   @require_detectrona
   @require_pytesseract
   def 			_lowerCAmelCase		( self     :		str			)       ->    List[str]:







      """simple docstring"""




      _UpperCAmelCase							:	Union[str, Any]							=   pipeline(
          "document-question-answering"						,     model="tiennvcs/layoutlmv2-base-uncased-finetuned-docvqa"						,     revision="9977165"						,     )
      _UpperCAmelCase							:	List[str]							=   INVOICE_URL
      _UpperCAmelCase							:	Any							=   "What is the invoice number?"

      _UpperCAmelCase							:	List[str]							=   dqa_pipeline(image=a__						,     question=a__						,     top_k=2			)
      self.assertEqual(
          nested_simplify(a__						,     decimals=4			)						,     [
              {"score": 0.9944, "answer": "us-001", "start": 1_6, "end": 1_6},
              {"score": 0.0009, "answer": "us-001", "start": 1_6, "end": 1_6},
          ]						,     )

      _UpperCAmelCase							:	Any							=   dqa_pipeline({"image": image, "question": question}						,     top_k=2			)
      self.assertEqual(
          nested_simplify(a__						,     decimals=4			)						,     [
              {"score": 0.9944, "answer": "us-001", "start": 1_6, "end": 1_6},
              {"score": 0.0009, "answer": "us-001", "start": 1_6, "end": 1_6},
          ]						,     )

      _UpperCAmelCase							:	Tuple							=   dqa_pipeline(
          [{"image": image, "question": question}, {"image": image, "question": question}]						,     top_k=2			)
      self.assertEqual(
          nested_simplify(a__						,     decimals=4			)						,     [
              [
                  {"score": 0.9944, "answer": "us-001", "start": 1_6, "end": 1_6},
                  {"score": 0.0009, "answer": "us-001", "start": 1_6, "end": 1_6},
              ],
          ]
          * 2						,     )




   @slow
   @require_torch
   @require_detectrona
   @require_pytesseract
   def 			_lowerCAmelCase		( self     :		List[Any]			)       ->    int:







      """simple docstring"""




      _UpperCAmelCase							:	int							=   pipeline(
          "document-question-answering"						,     model="tiennvcs/layoutlmv2-base-uncased-finetuned-docvqa"						,     revision="9977165"						,     max_seq_len=5_0						,     )
      _UpperCAmelCase							:	Any							=   INVOICE_URL
      _UpperCAmelCase							:	Tuple							=   "What is the invoice number?"

      _UpperCAmelCase							:	Optional[Any]							=   dqa_pipeline(image=a__						,     question=a__						,     top_k=2			)
      self.assertEqual(
          nested_simplify(a__						,     decimals=4			)						,     [
              {"score": 0.9974, "answer": "1110212019", "start": 2_3, "end": 2_3},
              {"score": 0.9948, "answer": "us-001", "start": 1_6, "end": 1_6},
          ]						,     )

      _UpperCAmelCase							:	Dict							=   dqa_pipeline({"image": image, "question": question}						,     top_k=2			)
      self.assertEqual(
          nested_simplify(a__						,     decimals=4			)						,     [
              {"score": 0.9974, "answer": "1110212019", "start": 2_3, "end": 2_3},
              {"score": 0.9948, "answer": "us-001", "start": 1_6, "end": 1_6},
          ]						,     )

      _UpperCAmelCase							:	Union[str, Any]							=   dqa_pipeline(
          [{"image": image, "question": question}, {"image": image, "question": question}]						,     top_k=2			)
      self.assertEqual(
          nested_simplify(a__						,     decimals=4			)						,     [
              [
                  {"score": 0.9974, "answer": "1110212019", "start": 2_3, "end": 2_3},
                  {"score": 0.9948, "answer": "us-001", "start": 1_6, "end": 1_6},
              ]
          ]
          * 2						,     )




   @slow
   @require_torch
   @require_pytesseract
   @require_vision
   def 			_lowerCAmelCase		( self     :		Tuple			)       ->    Union[str, Any]:







      """simple docstring"""




      _UpperCAmelCase							:	int							=   AutoTokenizer.from_pretrained(
          "impira/layoutlm-document-qa"						,     revision="3dc6de3"						,     add_prefix_space=a__			)
      _UpperCAmelCase							:	Any							=   pipeline(
          "document-question-answering"						,     model="impira/layoutlm-document-qa"						,     tokenizer=a__						,     revision="3dc6de3"						,     )
      _UpperCAmelCase							:	Any							=   INVOICE_URL
      _UpperCAmelCase							:	Optional[int]							=   "What is the invoice number?"

      _UpperCAmelCase							:	Union[str, Any]							=   dqa_pipeline(image=a__						,     question=a__						,     top_k=2			)
      self.assertEqual(
          nested_simplify(a__						,     decimals=4			)						,     [
              {"score": 0.4251, "answer": "us-001", "start": 1_6, "end": 1_6},
              {"score": 0.0819, "answer": "1110212019", "start": 2_3, "end": 2_3},
          ]						,     )

      _UpperCAmelCase							:	int							=   dqa_pipeline({"image": image, "question": question}						,     top_k=2			)
      self.assertEqual(
          nested_simplify(a__						,     decimals=4			)						,     [
              {"score": 0.4251, "answer": "us-001", "start": 1_6, "end": 1_6},
              {"score": 0.0819, "answer": "1110212019", "start": 2_3, "end": 2_3},
          ]						,     )

      _UpperCAmelCase							:	Optional[Any]							=   dqa_pipeline(
          [{"image": image, "question": question}, {"image": image, "question": question}]						,     top_k=2			)
      self.assertEqual(
          nested_simplify(a__						,     decimals=4			)						,     [
              [
                  {"score": 0.4251, "answer": "us-001", "start": 1_6, "end": 1_6},
                  {"score": 0.0819, "answer": "1110212019", "start": 2_3, "end": 2_3},
              ]
          ]
          * 2						,     )

      _UpperCAmelCase							:	Any							=   list(zip(*apply_tesseract(load_image(a__			)						,     a__						,     ""			)			)			)

      # This model should also work if `image` is set to None
      _UpperCAmelCase							:	Union[str, Any]							=   dqa_pipeline({"image": None, "word_boxes": word_boxes, "question": question}						,     top_k=2			)
      self.assertEqual(
          nested_simplify(a__						,     decimals=4			)						,     [
              {"score": 0.4251, "answer": "us-001", "start": 1_6, "end": 1_6},
              {"score": 0.0819, "answer": "1110212019", "start": 2_3, "end": 2_3},
          ]						,     )




   @slow
   @require_torch
   @require_pytesseract
   @require_vision
   def 			_lowerCAmelCase		( self     :		Dict			)       ->    Tuple:







      """simple docstring"""




      _UpperCAmelCase							:	Tuple							=   AutoTokenizer.from_pretrained(
          "impira/layoutlm-document-qa"						,     revision="3dc6de3"						,     add_prefix_space=a__			)
      _UpperCAmelCase							:	List[str]							=   pipeline(
          "document-question-answering"						,     model="impira/layoutlm-document-qa"						,     tokenizer=a__						,     revision="3dc6de3"						,     max_seq_len=5_0						,     )
      _UpperCAmelCase							:	Tuple							=   INVOICE_URL
      _UpperCAmelCase							:	int							=   "What is the invoice number?"

      _UpperCAmelCase							:	List[str]							=   dqa_pipeline(image=a__						,     question=a__						,     top_k=2			)
      self.assertEqual(
          nested_simplify(a__						,     decimals=4			)						,     [
              {"score": 0.9999, "answer": "us-001", "start": 1_6, "end": 1_6},
              {"score": 0.9998, "answer": "us-001", "start": 1_6, "end": 1_6},
          ]						,     )

      _UpperCAmelCase							:	List[Any]							=   dqa_pipeline(
          [{"image": image, "question": question}, {"image": image, "question": question}]						,     top_k=2			)
      self.assertEqual(
          nested_simplify(a__						,     decimals=4			)						,     [
              [
                  {"score": 0.9999, "answer": "us-001", "start": 1_6, "end": 1_6},
                  {"score": 0.9998, "answer": "us-001", "start": 1_6, "end": 1_6},
              ]
          ]
          * 2						,     )

      _UpperCAmelCase							:	Dict							=   list(zip(*apply_tesseract(load_image(a__			)						,     a__						,     ""			)			)			)

      # This model should also work if `image` is set to None
      _UpperCAmelCase							:	Tuple							=   dqa_pipeline({"image": None, "word_boxes": word_boxes, "question": question}						,     top_k=2			)
      self.assertEqual(
          nested_simplify(a__						,     decimals=4			)						,     [
              {"score": 0.9999, "answer": "us-001", "start": 1_6, "end": 1_6},
              {"score": 0.9998, "answer": "us-001", "start": 1_6, "end": 1_6},
          ]						,     )




   @slow
   @require_torch
   def 			_lowerCAmelCase		( self     :		Tuple			)       ->    int:







      """simple docstring"""




      _UpperCAmelCase							:	Union[str, Any]							=   pipeline(
          "document-question-answering"						,     model="naver-clova-ix/donut-base-finetuned-docvqa"						,     tokenizer=AutoTokenizer.from_pretrained("naver-clova-ix/donut-base-finetuned-docvqa"			)						,     feature_extractor="naver-clova-ix/donut-base-finetuned-docvqa"						,     )

      _UpperCAmelCase							:	Union[str, Any]							=   INVOICE_URL
      _UpperCAmelCase							:	Dict							=   "What is the invoice number?"
      _UpperCAmelCase							:	int							=   dqa_pipeline(image=a__						,     question=a__						,     top_k=2			)
      self.assertEqual(nested_simplify(a__						,     decimals=4			)						,     [{"answer": "us-001"}]			)




   @require_tf
   @unittest.skip("Document question answering not implemented in TF"			)
   def 			_lowerCAmelCase		( self     :		List[str]			)       ->    Optional[Any]:







      """simple docstring"""




      pass






import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin

@require_jieba
class 		SCREAMING_SNAKE_CASE__				(       _UpperCAmelCase						,      unittest.TestCase  ):
      A_						:				List[str]							=    CpmAntTokenizer
      A_						:				Optional[int]							=    False


      def    a (self       :      Optional[int]      ):





         """simple docstring"""



         super().setUp()

         __snake_case							       =   [
             '''<d>''',
             '''</d>''',
             '''<s>''',
             '''</s>''',
             '''</_>''',
             '''<unk>''',
             '''<pad>''',
             '''</n>''',
             '''我''',
             '''是''',
             '''C''',
             '''P''',
             '''M''',
             '''A''',
             '''n''',
             '''t''',
         ]
         __snake_case							       =   os.path.join(self.tmpdirname				,			VOCAB_FILES_NAMES['''vocab_file''']      )
         with open(self.vocab_file				,			'''w'''				,			encoding='''utf-8'''      ) as vocab_writer:
            vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]      )      )


      @tooslow
      def    a (self       :      Dict      ):





         """simple docstring"""



         __snake_case							       =   CpmAntTokenizer.from_pretrained('''openbmb/cpm-ant-10b'''      )
         __snake_case							       =   '''今天天气真好！'''
         __snake_case							       =   ['''今天''', '''天气''', '''真''', '''好''', '''！''']
         __snake_case							       =   tokenizer.tokenize(a__      )
         self.assertListEqual(a__				,			a__      )
         __snake_case							       =   '''今天天气真好！'''
         __snake_case							       =   [tokenizer.bos_token] + tokens

         __snake_case							       =   [6, 9802, 1_4962, 2082, 831, 244]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(a__      )				,			a__      )

         __snake_case							       =   tokenizer.decode(a__      )
         self.assertEqual(a__				,			a__      )

