
import inspect
import unittest

import requests

from transformers import MgpstrConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch
				from torch import nn

				from transformers import MgpstrForSceneTextRecognition


if is_vision_available():
				from PIL import Image

				from transformers import MgpstrProcessor






class 							SCREAMING_SNAKE_CASE__     :

							"""simple docstring"""






							def __init__(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_=False  ,						SCREAMING_SNAKE_CASE_=13  ,						SCREAMING_SNAKE_CASE_=(32, 128)  ,						SCREAMING_SNAKE_CASE_=4  ,						SCREAMING_SNAKE_CASE_=3  ,						SCREAMING_SNAKE_CASE_=27  ,						SCREAMING_SNAKE_CASE_=38  ,						SCREAMING_SNAKE_CASE_=99  ,						SCREAMING_SNAKE_CASE_=99  ,						SCREAMING_SNAKE_CASE_=32  ,						SCREAMING_SNAKE_CASE_=5  ,						SCREAMING_SNAKE_CASE_=4  ,						SCREAMING_SNAKE_CASE_=4.0  ,						SCREAMING_SNAKE_CASE_=257  ,						SCREAMING_SNAKE_CASE_=None  ,						)-> str:





													'''simple docstring'''






													__UpperCamelCase  	=      parent
													__UpperCamelCase  	=      is_training
													__UpperCamelCase  	=      batch_size
													__UpperCamelCase  	=      image_size
													__UpperCamelCase  	=      patch_size
													__UpperCamelCase  	=      num_channels
													__UpperCamelCase  	=      max_token_length
													__UpperCamelCase  	=      num_character_labels
													__UpperCamelCase  	=      num_bpe_labels
													__UpperCamelCase  	=      num_wordpiece_labels
													__UpperCamelCase  	=      hidden_size
													__UpperCamelCase  	=      num_hidden_layers
													__UpperCamelCase  	=      num_attention_heads
													__UpperCamelCase  	=      mlp_ratio
													__UpperCamelCase  	=      patch_embeds_hidden_size
													__UpperCamelCase  	=      output_hidden_states



							def      A__		(	self      )-> str:





													'''simple docstring'''






													__UpperCamelCase  	=      floats_tensor([self.batch_size, self.num_channels, self.image_size[0], self.image_size[1]]      )
													__UpperCamelCase  	=      self.get_config()
													return config, pixel_values



							def      A__		(	self      )-> Tuple:





													'''simple docstring'''






													return MgpstrConfig(
													    image_size=self.image_size  ,						patch_size=self.patch_size  ,						num_channels=self.num_channels  ,						max_token_length=self.max_token_length  ,						num_character_labels=self.num_character_labels  ,						num_bpe_labels=self.num_bpe_labels  ,						num_wordpiece_labels=self.num_wordpiece_labels  ,						hidden_size=self.hidden_size  ,						num_hidden_layers=self.num_hidden_layers  ,						num_attention_heads=self.num_attention_heads  ,						mlp_ratio=self.mlp_ratio  ,						output_hidden_states=self.output_hidden_states  ,						)



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_      )-> Dict:





													'''simple docstring'''






													__UpperCamelCase  	=      MgpstrForSceneTextRecognition(SCREAMING_SNAKE_CASE_      )
													model.to(SCREAMING_SNAKE_CASE_      )
													model.eval()
													with torch.no_grad():
																			__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_      )
													self.parent.assertEqual(
													    generated_ids[0][0].shape  ,						(self.batch_size, self.max_token_length, self.num_character_labels)      )



							def      A__		(	self      )-> Optional[int]:





													'''simple docstring'''






													__UpperCamelCase  	=      self.prepare_config_and_inputs()
													__UpperCamelCase	,				__UpperCamelCase  	=      config_and_inputs
													__UpperCamelCase  	=      {'''pixel_values''': pixel_values}
													return config, inputs_dict






@require_torch
class 							SCREAMING_SNAKE_CASE__     (				SCREAMING_SNAKE_CASE_ ,		SCREAMING_SNAKE_CASE_ ,		unittest.TestCase     ):

							"""simple docstring"""






							_snake_case  		=	(MgpstrForSceneTextRecognition,) if is_torch_available() else ()
							_snake_case  		=	{'feature-extraction': MgpstrForSceneTextRecognition} if is_torch_available() else {}
							_snake_case  		=	False

							_snake_case  		=	False
							_snake_case  		=	False
							_snake_case  		=	False
							_snake_case  		=	False



							def      A__		(	self      )-> Tuple:





													'''simple docstring'''






													__UpperCamelCase  	=      MgpstrModelTester(self      )
													__UpperCamelCase  	=      ConfigTester(self  ,						config_class=SCREAMING_SNAKE_CASE_  ,						has_text_modality=SCREAMING_SNAKE_CASE_      )



							def      A__		(	self      )-> Any:





													'''simple docstring'''






													self.config_tester.run_common_tests()



							def      A__		(	self      )-> List[Any]:





													'''simple docstring'''






													__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_      )



							@unittest.skip(reason='''MgpstrModel does not use inputs_embeds'''      )
							def      A__		(	self      )-> List[str]:





													'''simple docstring'''






													pass



							def      A__		(	self      )-> str:





													'''simple docstring'''






													__UpperCamelCase	,				__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs_for_common()

													for model_class in self.all_model_classes:
																			__UpperCamelCase  	=      model_class(SCREAMING_SNAKE_CASE_      )
																			self.assertIsInstance(model.get_input_embeddings()  ,						(nn.Module)      )
																			__UpperCamelCase  	=      model.get_output_embeddings()
																			self.assertTrue(x is None or isinstance(SCREAMING_SNAKE_CASE_  ,						nn.Linear      )      )



							def      A__		(	self      )-> List[Any]:





													'''simple docstring'''






													__UpperCamelCase	,				__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs_for_common()

													for model_class in self.all_model_classes:
																			__UpperCamelCase  	=      model_class(SCREAMING_SNAKE_CASE_      )
																			__UpperCamelCase  	=      inspect.signature(model.forward      )
																			# signature.parameters is an OrderedDict => so arg_names order is deterministic
																			__UpperCamelCase  	=      [*signature.parameters.keys()]

																			__UpperCamelCase  	=      ['''pixel_values''']
																			self.assertListEqual(arg_names[:1]  ,						SCREAMING_SNAKE_CASE_      )



							@unittest.skip(reason='''MgpstrModel does not support feedforward chunking'''      )
							def      A__		(	self      )-> Optional[Any]:





													'''simple docstring'''






													pass



							def      A__		(	self      )-> int:





													'''simple docstring'''






													__UpperCamelCase	,				__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs_for_common()

													for model_class in self.all_model_classes:
																			if not model_class.supports_gradient_checkpointing:
																									continue

																			__UpperCamelCase  	=      True
																			__UpperCamelCase  	=      model_class(SCREAMING_SNAKE_CASE_      )
																			self.assertTrue(model.is_gradient_checkpointing      )



							def      A__		(	self      )-> str:





													'''simple docstring'''






													def check_hidden_states_output(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_      ):
																			__UpperCamelCase  	=      model_class(SCREAMING_SNAKE_CASE_      )
																			model.to(SCREAMING_SNAKE_CASE_      )
																			model.eval()

																			with torch.no_grad():
																									__UpperCamelCase  	=      model(**self._prepare_for_class(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_      )      )

																			__UpperCamelCase  	=      outputs.hidden_states

																			__UpperCamelCase  	=      getattr(
																			    self.model_tester  ,						'''expected_num_hidden_layers'''  ,						self.model_tester.num_hidden_layers + 1      )
																			self.assertEqual(len(SCREAMING_SNAKE_CASE_      )  ,						SCREAMING_SNAKE_CASE_      )

																			self.assertListEqual(
																			    list(hidden_states[0].shape[-2:]      )  ,						[self.model_tester.patch_embeds_hidden_size, self.model_tester.hidden_size]  ,						)

													__UpperCamelCase	,				__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs_for_common()

													for model_class in self.all_model_classes:
																			__UpperCamelCase  	=      True
																			check_hidden_states_output(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_      )

																			# check that output_hidden_states also work using config
																			del inputs_dict["output_hidden_states"]
																			__UpperCamelCase  	=      True

																			check_hidden_states_output(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_      )



							def      A__		(	self      )-> Optional[int]:





													'''simple docstring'''






													__UpperCamelCase	,				__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs_for_common()

													__UpperCamelCase  	=      _config_zero_init(SCREAMING_SNAKE_CASE_      )
													for model_class in self.all_model_classes:
																			__UpperCamelCase  	=      model_class(config=SCREAMING_SNAKE_CASE_      )
																			for name, param in model.named_parameters():
																									if isinstance(SCREAMING_SNAKE_CASE_  ,						(nn.Linear, nn.Convad, nn.LayerNorm)      ):
																															if param.requires_grad:
																																					self.assertIn(
																																					    ((param.data.mean() * 1E9).round() / 1E9).item()  ,						[0.0, 1.0]  ,						msg=F"Parameter {name} of model {model_class} seems not properly initialized"  ,						)



							@unittest.skip(reason='''Retain_grad is tested in individual model tests'''      )
							def      A__		(	self      )-> List[Any]:





													'''simple docstring'''






													pass





def 						A_			(				)   ->     Dict:



						'''simple docstring'''

						__UpperCamelCase  	=      '''https://i.postimg.cc/ZKwLg2Gw/367-14.png'''
						__UpperCamelCase  	=      Image.open(requests.get(snake_case			,   stream=snake_case			).raw			).convert('''RGB'''			)
						return im






@require_vision
@require_torch
class 							SCREAMING_SNAKE_CASE__     (				unittest.TestCase     ):

							"""simple docstring"""






							@slow
							def      A__		(	self      )-> Optional[Any]:





													'''simple docstring'''






													__UpperCamelCase  	=      '''alibaba-damo/mgp-str-base'''
													__UpperCamelCase  	=      MgpstrForSceneTextRecognition.from_pretrained(SCREAMING_SNAKE_CASE_      ).to(SCREAMING_SNAKE_CASE_      )
													__UpperCamelCase  	=      MgpstrProcessor.from_pretrained(SCREAMING_SNAKE_CASE_      )

													__UpperCamelCase  	=      prepare_img()
													__UpperCamelCase  	=      processor(images=SCREAMING_SNAKE_CASE_  ,						return_tensors='''pt'''      ).pixel_values.to(SCREAMING_SNAKE_CASE_      )

													# forward pass
													with torch.no_grad():
																			__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_      )

													# verify the logits
													self.assertEqual(outputs.logits[0].shape  ,						torch.Size((1, 27, 38)      )      )

													__UpperCamelCase  	=      processor.batch_decode(outputs.logits      )
													__UpperCamelCase  	=      '''ticket'''

													self.assertEqual(out_strs['''generated_text'''][0]  ,						SCREAMING_SNAKE_CASE_      )

													__UpperCamelCase  	=      torch.tensor(
													    [[[-3_9.5_3_9_7, -4_4.4_0_2_4, -3_6.1_8_4_4], [-6_1.4_7_0_9, -6_3.8_6_3_9, -5_8.3_4_5_4], [-7_4.0_2_2_5, -6_8.5_4_9_4, -7_1.2_1_6_4]]]  ,						device=SCREAMING_SNAKE_CASE_  ,						)

													self.assertTrue(torch.allclose(outputs.logits[0][:, 1:4, 1:4]  ,						SCREAMING_SNAKE_CASE_  ,						atol=1E-4      )      )






import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_deberta import DebertaTokenizer


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation


UpperCAmelCase__						:	str	     = logging.get_logger(__name__)

UpperCAmelCase__						:	int	     = {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase__						:	int	     = {
    'vocab_file': {
        'microsoft/deberta-base': 'https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json',
        'microsoft/deberta-large': 'https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json',
        'microsoft/deberta-xlarge': 'https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json',
        'microsoft/deberta-base-mnli': 'https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json',
        'microsoft/deberta-large-mnli': 'https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json',
        'microsoft/deberta-xlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'microsoft/deberta-base': 'https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt',
        'microsoft/deberta-large': 'https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt',
        'microsoft/deberta-xlarge': 'https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt',
        'microsoft/deberta-base-mnli': 'https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt',
        'microsoft/deberta-large-mnli': 'https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt',
        'microsoft/deberta-xlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt'
        ),
    },
}

UpperCAmelCase__						:	List[str]	     = {
    'microsoft/deberta-base': 512,
    'microsoft/deberta-large': 512,
    'microsoft/deberta-xlarge': 512,
    'microsoft/deberta-base-mnli': 512,
    'microsoft/deberta-large-mnli': 512,
    'microsoft/deberta-xlarge-mnli': 512,
}

UpperCAmelCase__						:	Optional[Any]	     = {
    'microsoft/deberta-base': {'do_lower_case': False},
    'microsoft/deberta-large': {'do_lower_case': False},
}






class UpperCAmelCase  (		SCREAMING_SNAKE_CASE__  ):





   '''simple docstring'''
   __UpperCamelCase						:       Union[str, Any]												=       VOCAB_FILES_NAMES
   __UpperCamelCase						:       Union[str, Any]												=       PRETRAINED_VOCAB_FILES_MAP
   __UpperCamelCase						:       Union[str, Any]												=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCamelCase						:       str												=       ['''input_ids''', '''attention_mask''', '''token_type_ids''']
   __UpperCamelCase						:       Any												=       DebertaTokenizer


   def __init__(   self	:				Union[str, Any]     ,     lowerCAmelCase_	:				Optional[Any]=None     ,     lowerCAmelCase_	:				int=None     ,     lowerCAmelCase_	:				Dict=None     ,     lowerCAmelCase_	:				int="replace"     ,     lowerCAmelCase_	:				List[str]="[CLS]"     ,     lowerCAmelCase_	:				str="[SEP]"     ,     lowerCAmelCase_	:				int="[SEP]"     ,     lowerCAmelCase_	:				Any="[CLS]"     ,     lowerCAmelCase_	:				Dict="[UNK]"     ,     lowerCAmelCase_	:				Optional[int]="[PAD]"     ,     lowerCAmelCase_	:				List[str]="[MASK]"     ,     lowerCAmelCase_	:				Optional[Any]=False     ,     **lowerCAmelCase_	:				List[Any]     ,     ):

      """simple docstring"""
      super().__init__(
          lowerCAmelCase_     ,     lowerCAmelCase_     ,     tokenizer_file=lowerCAmelCase_     ,     errors=lowerCAmelCase_     ,     bos_token=lowerCAmelCase_     ,     eos_token=lowerCAmelCase_     ,     unk_token=lowerCAmelCase_     ,     sep_token=lowerCAmelCase_     ,     cls_token=lowerCAmelCase_     ,     pad_token=lowerCAmelCase_     ,     mask_token=lowerCAmelCase_     ,     add_prefix_space=lowerCAmelCase_     ,     **lowerCAmelCase_     ,     )
      _A:    List[str]						     =			kwargs.pop('''add_bos_token'''     ,     lowerCAmelCase_	)

      _A:    Optional[int]						     =			json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()	)
      if pre_tok_state.get('''add_prefix_space'''     ,     lowerCAmelCase_	) != add_prefix_space:
         _A:    Dict						     =			getattr(lowerCAmelCase_     ,     pre_tok_state.pop('''type'''	)	)
         _A:    List[Any]						     =			add_prefix_space
         _A:    Optional[int]						     =			pre_tok_class(**lowerCAmelCase_	)

      _A:    Optional[Any]						     =			add_prefix_space


   @property
   def 					__magic_name__							(   self	:				Optional[Any]	):

      """simple docstring"""
      if self._mask_token is None:
         if self.verbose:
            logger.error('''Using mask_token, but it is not set yet.'''	)
         return None
      return str(self._mask_token	)


   @mask_token.setter
   def 					__magic_name__							(   self	:				str     ,     lowerCAmelCase_	:				str	):

      """simple docstring"""
      _A:    Dict						     =			AddedToken(lowerCAmelCase_     ,     lstrip=lowerCAmelCase_     ,     rstrip=lowerCAmelCase_	) if isinstance(lowerCAmelCase_     ,     lowerCAmelCase_	) else value
      _A:    str						     =			value


   def 					__magic_name__							(   self	:				Optional[Any]     ,     lowerCAmelCase_	:				List[int]     ,     lowerCAmelCase_	:				Optional[List[int]] = None	):

      """simple docstring"""
      if token_ids_a is None:
         return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      _A:    str						     =			[self.cls_token_id]
      _A:    Any						     =			[self.sep_token_id]
      return cls + token_ids_a + sep + token_ids_a + sep


   def 					__magic_name__							(   self	:				Optional[int]     ,     lowerCAmelCase_	:				List[int]     ,     lowerCAmelCase_	:				Optional[List[int]] = None	):

      """simple docstring"""
      _A:    Dict						     =			[self.sep_token_id]
      _A:    Any						     =			[self.cls_token_id]

      if token_ids_a is None:
         return len(cls + token_ids_a + sep	) * [0]
      return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]


   def 					__magic_name__							(   self	:				Tuple     ,     *lowerCAmelCase_	:				Dict     ,     **lowerCAmelCase_	:				Tuple	):

      """simple docstring"""
      _A:    Any						     =			kwargs.get('''is_split_into_words'''     ,     lowerCAmelCase_	)
      assert self.add_prefix_space or not is_split_into_words, (
          F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
          "to use it with pretokenized inputs."
      )

      return super()._batch_encode_plus(*lowerCAmelCase_     ,     **lowerCAmelCase_	)


   def 					__magic_name__							(   self	:				List[Any]     ,     *lowerCAmelCase_	:				Optional[Any]     ,     **lowerCAmelCase_	:				Any	):

      """simple docstring"""
      _A:    Optional[Any]						     =			kwargs.get('''is_split_into_words'''     ,     lowerCAmelCase_	)

      assert self.add_prefix_space or not is_split_into_words, (
          F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
          "to use it with pretokenized inputs."
      )

      return super()._encode_plus(*lowerCAmelCase_     ,     **lowerCAmelCase_	)


   def 					__magic_name__							(   self	:				Optional[int]     ,     lowerCAmelCase_	:				str     ,     lowerCAmelCase_	:				Optional[str] = None	):

      """simple docstring"""
      _A:    Union[str, Any]						     =			self._tokenizer.model.save(lowerCAmelCase_     ,     name=lowerCAmelCase_	)
      return tuple(lowerCAmelCase_	)


   def 					__magic_name__							(   self	:				Union[str, Any]     ,     lowerCAmelCase_	:				"Conversation"	):

      """simple docstring"""
      _A:    List[Any]						     =			[]
      for is_user, text in conversation.iter_texts():
         input_ids.extend(self.encode(lowerCAmelCase_     ,     add_special_tokens=lowerCAmelCase_	) + [self.eos_token_id]	)

      if len(lowerCAmelCase_	) > self.model_max_length:
         _A:    Union[str, Any]						     =			input_ids[-self.model_max_length :]
      return input_ids



