




"""simple docstring"""






import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase__     						=						logging.get_logger(__name__)

lowercase__     						=						"""▁"""

lowercase__     						=						{"""vocab_file""": """vocab.txt""", """sentencepiece_model_ckpt""": """sentencepiece.bpe.model"""}

lowercase__     						=						{
    """sentencepiece_model_file""": """sentencepiece.bpe.model""",
    """vocab_file""": """vocab.txt""",
}

lowercase__     						=						{
    """vocab_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
    },
    """sentencepiece_model_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
    },
}

lowercase__     						=						{
    """ernie-m-base""": 514,
    """ernie-m-large""": 514,
}

lowercase__     						=						{
    """ernie-m-base""": {"""do_lower_case""": False},
    """ernie-m-large""": {"""do_lower_case""": False},
}






class lowerCAmelCase__	(     lowercase						):






			'''simple docstring'''



			lowerCamelCase__      =			["input_ids"]

			lowerCamelCase__      =			VOCAB_FILES_NAMES
			lowerCamelCase__      =			PRETRAINED_INIT_CONFIGURATION
			lowerCamelCase__      =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase__      =			PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase__      =			RESOURCE_FILES_NAMES
			def __init__(  self  ,      lowercase  ,      lowercase=None  ,      lowercase=False  ,      lowercase="utf8"  ,      lowercase="[UNK]"  ,      lowercase="[SEP]"  ,      lowercase="[PAD]"  ,      lowercase="[CLS]"  ,      lowercase="[MASK]"  ,      lowercase = None  ,      **lowercase  ,      ):
							# Mask token behave like a normal word, i.e. include the space before it and
							# is included in the raw text, there should be a match in a non-normalized sentence.

							_lowerCamelCase     :			Optional[int]    =    {} if sp_model_kwargs is None else sp_model_kwargs
							super().__init__(
							    do_lower_case=lowercase  ,      unk_token=lowercase  ,      sep_token=lowercase  ,      pad_token=lowercase  ,      cls_token=lowercase  ,      mask_token=lowercase  ,      vocab_file=lowercase  ,      encoding=lowercase  ,      sp_model_kwargs=self.sp_model_kwargs  ,      **lowercase  ,      )
							_lowerCamelCase     :			str    =    do_lower_case
							_lowerCamelCase     :			Optional[Any]    =    sentencepiece_model_ckpt
							_lowerCamelCase     :			int    =    spm.SentencePieceProcessor(**self.sp_model_kwargs  )
							self.sp_model.Load(lowercase  )

							# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
							if vocab_file is not None:
											_lowerCamelCase     :			Dict    =    self.load_vocab(filepath=lowercase  )
							else:
											_lowerCamelCase     :			Optional[int]    =    {self.sp_model.id_to_piece(lowercase  ): id for id in range(self.sp_model.get_piece_size()  )}
							_lowerCamelCase     :			int    =    {v: k for k, v in self.vocab.items()}
			def 					A_    (  self  ,      lowercase  ):
							if text is None:
											return None

							_lowerCamelCase     :			Tuple    =    self.tokenize(lowercase  )
							_lowerCamelCase, _lowerCamelCase     :			Any    =    '', []

							for i, ch in enumerate(lowercase  ):
											if ch in self.SP_CHAR_MAPPING:
															_lowerCamelCase     :			List[str]    =    self.SP_CHAR_MAPPING.get(lowercase  )
											else:
															_lowerCamelCase     :			Dict    =    unicodedata.normalize('NFKC'  ,      lowercase  )
											if self.is_whitespace(lowercase  ):
															continue
											normalized_text += ch
											char_mapping.extend([i] * len(lowercase  )  )

							_lowerCamelCase, _lowerCamelCase, _lowerCamelCase     :			Union[str, Any]    =    normalized_text, [], 0

							if self.do_lower_case:
											_lowerCamelCase     :			List[str]    =    text.lower()

							for token in split_tokens:
											if token[:1] == "▁":
															_lowerCamelCase     :			List[str]    =    token[1:]
											_lowerCamelCase     :			Union[str, Any]    =    text[offset:].index(lowercase  ) + offset
											_lowerCamelCase     :			Optional[int]    =    start + len(lowercase  )

											token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)  )
											_lowerCamelCase     :			Optional[Any]    =    end
							return token_mapping
			@property
			def 					A_    (  self  ):
							return len(self.vocab  )
			def 					A_    (  self  ):
							return dict(self.vocab  ,      **self.added_tokens_encoder  )
			def __getstate__(  self  ):
							_lowerCamelCase     :			str    =    self.__dict__.copy()
							_lowerCamelCase     :			Optional[int]    =    None
							return state
			def __setstate__(  self  ,      lowercase  ):
							_lowerCamelCase     :			Union[str, Any]    =    d

							# for backward compatibility
							if not hasattr(self  ,      'sp_model_kwargs'  ):
											_lowerCamelCase     :			List[str]    =    {}

							_lowerCamelCase     :			str    =    spm.SentencePieceProcessor(**self.sp_model_kwargs  )
							self.sp_model.Load(self.sentencepiece_model_ckpt  )
			def 					A_    (  self  ,      lowercase  ):
							return "".join((self.SP_CHAR_MAPPING.get(lowercase  ,      lowercase  ) for c in text)  )
			def 					A_    (  self  ,      lowercase  ,      lowercase=False  ,      lowercase=64  ,      lowercase=0.1  ):

							if self.sp_model_kwargs.get('enable_sampling'  ) is True:
											_lowerCamelCase     :			Optional[int]    =    True
							if self.sp_model_kwargs.get('alpha'  ) is not None:
											_lowerCamelCase     :			Tuple    =    self.sp_model_kwargs.get('alpha'  )
							if self.sp_model_kwargs.get('nbest_size'  ) is not None:
											_lowerCamelCase     :			Tuple    =    self.sp_model_kwargs.get('nbest_size'  )

							if not enable_sampling:
											_lowerCamelCase     :			Optional[Any]    =    self.sp_model.EncodeAsPieces(lowercase  )
							else:
											_lowerCamelCase     :			List[str]    =    self.sp_model.SampleEncodeAsPieces(lowercase  ,      lowercase  ,      lowercase  )
							_lowerCamelCase     :			Tuple    =    []
							for pi, piece in enumerate(lowercase  ):
											if piece == SPIECE_UNDERLINE:
															if not pieces[pi + 1].startswith(lowercase  ) and pi != 0:
																			new_pieces.append(lowercase  )
																			continue
															else:
																			continue
											_lowerCamelCase     :			Tuple    =    0
											for i, chunk in enumerate(lowercase  ):
															if chunk == SPIECE_UNDERLINE:
																			continue
															if self.is_ch_char(lowercase  ) or self.is_punct(lowercase  ):
																			if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																							new_pieces.append(piece[lst_i:i]  )
																			new_pieces.append(lowercase  )
																			_lowerCamelCase     :			int    =    i + 1
															elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																			if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																							new_pieces.append(piece[lst_i:i]  )
																			_lowerCamelCase     :			Any    =    i
															elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																			if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																							new_pieces.append(piece[lst_i:i]  )
																			_lowerCamelCase     :			Tuple    =    i
											if len(lowercase  ) > lst_i:
															new_pieces.append(piece[lst_i:]  )
							return new_pieces
			def 					A_    (  self  ,      lowercase  ):
							_lowerCamelCase     :			Optional[int]    =    ''.join(lowercase  ).replace(lowercase  ,      ' '  ).strip()
							return out_string
			def 					A_    (  self  ,      lowercase  ):
							_lowerCamelCase     :			Union[str, Any]    =    self.convert_ids_to_tokens(lowercase  )
							_lowerCamelCase     :			Any    =    ''.join(lowercase  ).replace(lowercase  ,      ' '  ).strip()
							return out_string
			def 					A_    (  self  ,      lowercase  ):
							return self.vocab.get(lowercase  ,      self.vocab.get(self.unk_token  )  )
			def 					A_    (  self  ,      lowercase  ):
							return self.reverse_vocab.get(lowercase  ,      self.unk_token  )
			def 					A_    (  self  ,      lowercase  ,      lowercase=None  ):
							if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							_lowerCamelCase     :			List[Any]    =    [self.cls_token_id]
							_lowerCamelCase     :			Optional[Any]    =    [self.sep_token_id]
							return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep
			def 					A_    (  self  ,      lowercase  ,      lowercase=None  ):
							if offset_mapping_a is None:
											return [(0, 0)] + offset_mapping_a + [(0, 0)]

							return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]
			def 					A_    (  self  ,      lowercase  ,      lowercase=None  ,      lowercase=False  ):

							if already_has_special_tokens:
											if token_ids_a is not None:
															raise ValueError(
															    'You should not supply a second sequence if the provided sequence of '
															    'ids is already formatted with special tokens for the model.'  )
											return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

							if token_ids_a is not None:
											return [1] + ([0] * len(lowercase  )) + [1, 1] + ([0] * len(lowercase  )) + [1]
							return [1] + ([0] * len(lowercase  )) + [1]
			def 					A_    (  self  ,      lowercase  ,      lowercase = None  ):
							# called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
							if token_ids_a is None:
											# [CLS] X [SEP]
											return (len(lowercase  ) + 2) * [0]

							# [CLS] A [SEP] [SEP] B [SEP]
							return [0] * (len(lowercase  ) + 1) + [1] * (len(lowercase  ) + 3)
			def 					A_    (  self  ,      lowercase  ):
							if "\u4e00" <= char <= "\u9fff":
											return True
							return False
			def 					A_    (  self  ,      lowercase  ):
							if ("a" <= char <= "z") or ("A" <= char <= "Z"):
											return True
							return False
			def 					A_    (  self  ,      lowercase  ):
							if char in ",;:.?!~，；：。？！《》【】":
											return True
							return False
			def 					A_    (  self  ,      lowercase  ):
							if char == " " or char == "\t" or char == "\n" or char == "\r":
											return True
							if len(lowercase  ) == 1:
											_lowerCamelCase     :			Tuple    =    unicodedata.category(lowercase  )
											if cat == "Zs":
															return True
							return False
			def 					A_    (  self  ,      lowercase  ):
							_lowerCamelCase     :			Tuple    =    {}
							with io.open(lowercase  ,      'r'  ,      encoding='utf-8'  ) as f:
											for index, line in enumerate(lowercase  ):
															_lowerCamelCase     :			int    =    line.rstrip('\n'  )
															_lowerCamelCase     :			Optional[Any]    =    int(lowercase  )

							return token_to_idx






			def 					A_    (  self  ,      lowercase  ,      lowercase = None  ):
							_lowerCamelCase     :			Union[str, Any]    =    0
							if os.path.isdir(lowercase  ):
											_lowerCamelCase     :			List[Any]    =    os.path.join(
											    lowercase  ,      (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )
							else:
											_lowerCamelCase     :			Optional[int]    =    (filename_prefix + '-' if filename_prefix else '') + save_directory
							with open(lowercase  ,      'w'  ,      encoding='utf-8'  ) as writer:
											for token, token_index in sorted(self.vocab.items()  ,      key=lambda lowercase  : kv[1]  ):
															if index != token_index:
																			logger.warning(
																			    F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
																			    ' Please check that the vocabulary is not corrupted!'  )
																			_lowerCamelCase     :			Optional[Any]    =    token_index
															writer.write(token + '\n'  )
															index += 1

							_lowerCamelCase     :			List[str]    =    os.path.join(lowercase  ,      'sentencepiece.bpe.model'  )
							with open(lowercase  ,      'wb'  ) as fi:
											_lowerCamelCase     :			Dict    =    self.sp_model.serialized_model_proto()
											fi.write(lowercase  )

							return (vocab_file,)



'''simple docstring'''

import unittest

from transformers import TrOCRConfig
from transformers.testing_utils import is_torch_available, require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers.models.trocr.modeling_trocr import TrOCRDecoder, TrOCRForCausalLM
@require_torch
class   __lowerCAmelCase						:





      '''simple docstring'''






      def __init__(self    :    str   ,						UpperCamelCase    :    Tuple   ,						UpperCamelCase    :    Optional[int]=99   ,						UpperCamelCase    :    Optional[int]=13   ,						UpperCamelCase    :    Tuple=16   ,						UpperCamelCase    :    Union[str, Any]=7   ,						UpperCamelCase    :    List[Any]=True   ,						UpperCamelCase    :    List[str]=True   ,						UpperCamelCase    :    str=True   ,						UpperCamelCase    :    Tuple=False   ,						UpperCamelCase    :    str=True   ,						UpperCamelCase    :    Tuple=2   ,						UpperCamelCase    :    Optional[int]=32   ,						UpperCamelCase    :    Any=4   ,						UpperCamelCase    :    Optional[int]=4   ,						UpperCamelCase    :    Tuple=30   ,						UpperCamelCase    :    str=0   ,						UpperCamelCase    :    Tuple=1   ,						UpperCamelCase    :    List[Any]=2   ,						UpperCamelCase    :    str=None   ,						):

             '''simple docstring'''







             lowercase__      			= parent
             lowercase__      			= batch_size
             lowercase__      			= decoder_seq_length
             # For common tests
             lowercase__      			= self.decoder_seq_length
             lowercase__      			= is_training
             lowercase__      			= use_attention_mask
             lowercase__      			= use_labels

             lowercase__      			= vocab_size
             lowercase__      			= d_model
             lowercase__      			= d_model
             lowercase__      			= decoder_layers
             lowercase__      			= decoder_layers
             lowercase__      			= decoder_ffn_dim
             lowercase__      			= decoder_attention_heads
             lowercase__      			= decoder_attention_heads
             lowercase__      			= eos_token_id
             lowercase__      			= bos_token_id
             lowercase__      			= pad_token_id
             lowercase__      			= decoder_start_token_id
             lowercase__      			= use_cache
             lowercase__      			= max_position_embeddings

             lowercase__      			= None
             lowercase__      			= decoder_seq_length
             lowercase__      			= 2
             lowercase__      			= 1



      def       UpperCamelCase__		(self    :    str ):

             '''simple docstring'''







             lowercase__      			= ids_tensor([self.batch_size, self.decoder_seq_length]   ,						self.vocab_size )

             lowercase__      			= None
             if self.use_attention_mask:
                    lowercase__      			= ids_tensor([self.batch_size, self.decoder_seq_length]   ,						vocab_size=2 )

             lowercase__      			= None
             if self.use_labels:
                    lowercase__      			= ids_tensor([self.batch_size, self.decoder_seq_length]   ,						self.vocab_size )

             lowercase__      			= TrOCRConfig(
                 vocab_size=self.vocab_size   ,						d_model=self.d_model   ,						decoder_layers=self.decoder_layers   ,						decoder_ffn_dim=self.decoder_ffn_dim   ,						decoder_attention_heads=self.decoder_attention_heads   ,						eos_token_id=self.eos_token_id   ,						bos_token_id=self.bos_token_id   ,						use_cache=self.use_cache   ,						pad_token_id=self.pad_token_id   ,						decoder_start_token_id=self.decoder_start_token_id   ,						max_position_embeddings=self.max_position_embeddings   ,						)

             return (config, input_ids, attention_mask, lm_labels)



      def       UpperCamelCase__		(self    :    Tuple   ,						UpperCamelCase    :    List[Any]   ,						UpperCamelCase    :    Optional[Any]   ,						UpperCamelCase    :    Tuple   ,						UpperCamelCase    :    List[str]   ,						):

             '''simple docstring'''







             lowercase__      			= True
             lowercase__      			= TrOCRDecoder(config=UpperCamelCase ).to(UpperCamelCase ).eval()
             lowercase__      			= input_ids[:2]

             input_ids[input_ids == 0] += 1
             # first forward pass
             lowercase__      			= model(UpperCamelCase   ,						use_cache=UpperCamelCase )
             lowercase__      			= model(UpperCamelCase )
             lowercase__      			= model(UpperCamelCase   ,						use_cache=UpperCamelCase )

             self.parent.assertTrue(len(UpperCamelCase ) == len(UpperCamelCase ) )
             self.parent.assertTrue(len(UpperCamelCase ) == len(UpperCamelCase ) + 1 )

             lowercase__      			= outputs['''past_key_values''']

             # create hypothetical next token and extent to next_input_ids
             lowercase__      			= ids_tensor((2, 1)   ,						config.vocab_size - 1 ) + 1

             # append to next input_ids and
             lowercase__      			= torch.cat([input_ids, next_tokens]   ,						dim=-1 )

             lowercase__      			= model(UpperCamelCase )['''last_hidden_state''']
             lowercase__      			= model(UpperCamelCase   ,						past_key_values=UpperCamelCase )['''last_hidden_state''']

             # select random slice
             lowercase__      			= ids_tensor((1,)   ,						output_from_past.shape[-1] ).item()
             lowercase__      			= output_from_no_past[:, next_input_ids.shape[-1] - 1, random_slice_idx].detach()
             lowercase__      			= output_from_past[:, 0, random_slice_idx].detach()

             # test that outputs are equal for slice
             assert torch.allclose(UpperCamelCase   ,						UpperCamelCase   ,						atol=1E-3 )






      def       UpperCamelCase__		(self    :    Optional[Any] ):

             '''simple docstring'''







             lowercase__      			= self.prepare_config_and_inputs()
             lowercase__			,lowercase__			,lowercase__			,lowercase__      			= config_and_inputs

             lowercase__      			= {'''input_ids''': input_ids, '''attention_mask''': attention_mask}
             return config, inputs_dict






@require_torch
class   __lowerCAmelCase						(lowercase_  ,     lowercase_  ,     lowercase_  ,     unittest.TestCase     ):





      '''simple docstring'''






      lowerCAmelCase__					:     List[str]  					=							(TrOCRDecoder, TrOCRForCausalLM) if is_torch_available() else ()
      lowerCAmelCase__					:     List[Any]  					=							(TrOCRForCausalLM,) if is_torch_available() else ()
      lowerCAmelCase__					:     Optional[Any]  					=							{"""text-generation""": TrOCRForCausalLM} if is_torch_available() else {}
      lowerCAmelCase__					:     Optional[Any]  					=							True
      lowerCAmelCase__					:     List[str]  					=							False



      def       UpperCamelCase__		(self    :    Any ):

             '''simple docstring'''







             lowercase__      			= TrOCRStandaloneDecoderModelTester(self   ,						is_training=UpperCamelCase )
             lowercase__      			= ConfigTester(self   ,						config_class=UpperCamelCase )



      def       UpperCamelCase__		(self    :    List[str] ):

             '''simple docstring'''







             pass



      def       UpperCamelCase__		(self    :    Optional[int] ):

             '''simple docstring'''







             pass



      def       UpperCamelCase__		(self    :    Any ):

             '''simple docstring'''







             pass



      def       UpperCamelCase__		(self    :    Any ):

             '''simple docstring'''







             self.config_tester.run_common_tests()



      def       UpperCamelCase__		(self    :    Union[str, Any] ):

             '''simple docstring'''







             lowercase__      			= self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_decoder_model_past(*UpperCamelCase )



      def       UpperCamelCase__		(self    :    Optional[int] ):

             '''simple docstring'''







             return






      @unittest.skip('''The model doesn\'t support left padding''' )  # and it's not used enough to be worth fixing :)
      def       UpperCamelCase__		(self    :    List[str] ):

             '''simple docstring'''







             pass




