



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


A__  :  int								=					logging.get_logger(__name__)

A__  :  List[Any]								=					'''▁'''

A__  :  Union[str, Any]								=					{'''vocab_file''': '''sentencepiece.bpe.model'''}

A__  :  Any								=					{
    '''vocab_file''': {
        '''facebook/mbart-large-en-ro''': (
            '''https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'''
        ),
        '''facebook/mbart-large-cc25''': (
            '''https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'''
        ),
    }
}

A__  :  List[Any]								=					{
    '''facebook/mbart-large-en-ro''': 1024,
    '''facebook/mbart-large-cc25''': 1024,
}

# fmt: off
A__  :  Any								=					['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''']



class 				__snake_case						(     UpperCamelCase_						):
							_a			      = VOCAB_FILES_NAMES
							_a			      = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_a			      = PRETRAINED_VOCAB_FILES_MAP
							_a			      = ['''input_ids''', '''attention_mask''']

							_a			      = []
							_a			      = []




							def __init__(       self     :       str				,				A_     :       Any				,				A_     :       Optional[Any]="<s>"				,				A_     :       int="</s>"				,				A_     :       Dict="</s>"				,				A_     :       int="<s>"				,				A_     :       Any="<unk>"				,				A_     :       Optional[Any]="<pad>"				,				A_     :       Optional[int]="<mask>"				,				A_     :       Optional[Any]=None				,				A_     :       List[str]=None				,				A_     :       List[str]=None				,				A_     :       Optional[Dict[str, Any]] = None				,				A_     :       Any=None				,				**A_     :       List[Any]				,				):
									# Mask token behave like a normal word, i.e. include the space before it
									lowerCAmelCase_			:     int									= AddedToken(A_				,				lstrip=A_				,				rstrip=A_) if isinstance(A_				,				A_) else mask_token

									lowerCAmelCase_			:     Dict									= {} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    bos_token=A_				,				eos_token=A_				,				unk_token=A_				,				sep_token=A_				,				cls_token=A_				,				pad_token=A_				,				mask_token=A_				,				tokenizer_file=A_				,				src_lang=A_				,				tgt_lang=A_				,				additional_special_tokens=A_				,				sp_model_kwargs=self.sp_model_kwargs				,				**A_				,				)

									lowerCAmelCase_			:     Dict									= spm.SentencePieceProcessor(**self.sp_model_kwargs)
									self.sp_model.Load(str(A_))
									lowerCAmelCase_			:     List[Any]									= vocab_file

									# Original fairseq vocab and spm vocab must be "aligned":
									# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
									# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
									# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
									# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

									# Mimic fairseq token-to-id alignment for the first 4 token
									lowerCAmelCase_			:     str									= {'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

									# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
									lowerCAmelCase_			:     Tuple									= 1

									lowerCAmelCase_			:     Union[str, Any]									= len(self.sp_model)
									lowerCAmelCase_			:     Any									= {
									    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(A_)
									}
									lowerCAmelCase_			:     Optional[int]									= {v: k for k, v in self.lang_code_to_id.items()}
									lowerCAmelCase_			:     Union[str, Any]									= len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset

									self.fairseq_tokens_to_ids.update(self.lang_code_to_id)
									lowerCAmelCase_			:     Dict									= {v: k for k, v in self.fairseq_tokens_to_ids.items()}
									lowerCAmelCase_			:     Union[str, Any]									= list(self.lang_code_to_id.keys())

									if additional_special_tokens is not None:
											# Only add those special tokens if they are not already there.
											self._additional_special_tokens.extend(
											    [t for t in additional_special_tokens if t not in self._additional_special_tokens])

									lowerCAmelCase_			:     Dict									= src_lang if src_lang is not None else '''en_XX'''
									lowerCAmelCase_			:     int									= self.lang_code_to_id[self._src_lang]
									lowerCAmelCase_			:     List[Any]									= tgt_lang
									self.set_src_lang_special_tokens(self._src_lang)




							def __getstate__(       self     :       Optional[Any]):
									lowerCAmelCase_			:     Dict									= self.__dict__.copy()
									lowerCAmelCase_			:     Optional[int]									= None
									lowerCAmelCase_			:     Optional[int]									= self.sp_model.serialized_model_proto()
									return state




							def __setstate__(       self     :       Optional[int]				,				A_     :       Optional[int]):
									lowerCAmelCase_			:     str									= d

									# for backward compatibility
									if not hasattr(self				,				'''sp_model_kwargs'''):
											lowerCAmelCase_			:     List[Any]									= {}

									lowerCAmelCase_			:     List[str]									= spm.SentencePieceProcessor(**self.sp_model_kwargs)
									self.sp_model.LoadFromSerializedProto(self.sp_model_proto)




							@property
							def    UpperCAmelCase__			(       self     :       Tuple):
									return len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset + 1  # Plus 1 for the mask token




							@property
							def    UpperCAmelCase__			(       self     :       int):
									return self._src_lang




							@src_lang.setter
							def    UpperCAmelCase__			(       self     :       List[Any]				,				A_     :       str):
									lowerCAmelCase_			:     List[str]									= new_src_lang
									self.set_src_lang_special_tokens(self._src_lang)




							def    UpperCAmelCase__			(       self     :       List[str]				,				A_     :       List[int]				,				A_     :       Optional[List[int]] = None				,				A_     :       bool = False):

									if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=A_				,				token_ids_a=A_				,				already_has_special_tokens=A_)

									lowerCAmelCase_			:     Dict									= [1] * len(self.prefix_tokens)
									lowerCAmelCase_			:     Optional[int]									= [1] * len(self.suffix_tokens)
									if token_ids_a is None:
											return prefix_ones + ([0] * len(A_)) + suffix_ones
									return prefix_ones + ([0] * len(A_)) + ([0] * len(A_)) + suffix_ones




							def    UpperCAmelCase__			(       self     :       Optional[int]				,				A_     :       List[int]				,				A_     :       Optional[List[int]] = None):
									if token_ids_a is None:
											return self.prefix_tokens + token_ids_a + self.suffix_tokens
									# We don't expect to process pairs, but leave the pair logic for API consistency
									return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




							def    UpperCAmelCase__			(       self     :       Tuple				,				A_     :       List[int]				,				A_     :       Optional[List[int]] = None):
									lowerCAmelCase_			:     Union[str, Any]									= [self.sep_token_id]
									lowerCAmelCase_			:     Optional[Any]									= [self.cls_token_id]

									if token_ids_a is None:
											return len(cls + token_ids_a + sep) * [0]
									return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]




							def    UpperCAmelCase__			(       self     :       Any				,				A_     :       int				,				A_     :       str				,				A_     :       Optional[str]				,				A_     :       Optional[str]				,				**A_     :       Optional[Any]):
									if src_lang is None or tgt_lang is None:
											raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model''')
									lowerCAmelCase_			:     str									= src_lang
									lowerCAmelCase_			:     Tuple									= self(A_				,				add_special_tokens=A_				,				return_tensors=A_				,				**A_)
									lowerCAmelCase_			:     Any									= self.convert_tokens_to_ids(A_)
									lowerCAmelCase_			:     Optional[Any]									= tgt_lang_id
									return inputs




							def    UpperCAmelCase__			(       self     :       Dict):
									lowerCAmelCase_			:     Optional[Any]									= {self.convert_ids_to_tokens(A_): i for i in range(self.vocab_size)}
									vocab.update(self.added_tokens_encoder)
									return vocab




							def    UpperCAmelCase__			(       self     :       List[Any]				,				A_     :       str):
									return self.sp_model.encode(A_				,				out_type=A_)




							def    UpperCAmelCase__			(       self     :       List[Any]				,				A_     :       Optional[Any]):
									if token in self.fairseq_tokens_to_ids:
											return self.fairseq_tokens_to_ids[token]
									lowerCAmelCase_			:     Any									= self.sp_model.PieceToId(A_)

									# Need to return unknown token if the SP model returned 0
									return spm_id + self.fairseq_offset if spm_id else self.unk_token_id




							def    UpperCAmelCase__			(       self     :       int				,				A_     :       int):
									if index in self.fairseq_ids_to_tokens:
											return self.fairseq_ids_to_tokens[index]
									return self.sp_model.IdToPiece(index - self.fairseq_offset)




							def    UpperCAmelCase__			(       self     :       Dict				,				A_     :       int):
									lowerCAmelCase_			:     Union[str, Any]									= ''''''.join(A_).replace(A_				,				''' ''').strip()
									return out_string




							def    UpperCAmelCase__			(       self     :       List[Any]				,				A_     :       str				,				A_     :       Optional[str] = None):
									if not os.path.isdir(A_):
											logger.error(F"""Vocabulary path ({save_directory}) should be a directory""")
											return
									lowerCAmelCase_			:     Optional[Any]									= os.path.join(
									    A_				,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

									if os.path.abspath(self.vocab_file) != os.path.abspath(A_) and os.path.isfile(self.vocab_file):
											copyfile(self.vocab_file				,				A_)
									elif not os.path.isfile(self.vocab_file):
											with open(A_				,				'''wb''') as fi:
													lowerCAmelCase_			:     Union[str, Any]									= self.sp_model.serialized_model_proto()
													fi.write(A_)

									return (out_vocab_file,)




							def    UpperCAmelCase__			(       self     :       List[Any]				,				A_     :       List[str]				,				A_     :       str = "en_XX"				,				A_     :       Optional[List[str]] = None				,				A_     :       str = "ro_RO"				,				**A_     :       Tuple				,				):
									lowerCAmelCase_			:     int									= src_lang
									lowerCAmelCase_			:     Union[str, Any]									= tgt_lang
									return super().prepare_seqaseq_batch(A_				,				A_				,				**A_)




							def    UpperCAmelCase__			(       self     :       List[str]):
									return self.set_src_lang_special_tokens(self.src_lang)




							def    UpperCAmelCase__			(       self     :       List[str]):
									return self.set_tgt_lang_special_tokens(self.tgt_lang)




							def    UpperCAmelCase__			(       self     :       Union[str, Any]				,				A_     :       Union[str, Any]):
									lowerCAmelCase_			:     List[Any]									= self.lang_code_to_id[src_lang]
									lowerCAmelCase_			:     Dict									= []
									lowerCAmelCase_			:     Tuple									= [self.eos_token_id, self.cur_lang_code]







							def    UpperCAmelCase__			(       self     :       Optional[Any]				,				A_     :       str):
									lowerCAmelCase_			:     List[Any]									= self.lang_code_to_id[lang]
									lowerCAmelCase_			:     List[Any]									= []
									lowerCAmelCase_			:     Dict									= [self.eos_token_id, self.cur_lang_code]












'''simple docstring'''







from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase_					:    Optional[Any]      	=			logging.get_logger(__name__)

lowerCAmelCase_					:    List[str]      	=			{
    'microsoft/trocr-base-handwritten': (
        'https://huggingface.co/microsoft/trocr-base-handwritten/resolve/main/config.json'
    ),
    # See all TrOCR models at https://huggingface.co/models?filter=trocr
}
class 				__SCREAMING_SNAKE_CASE  (lowerCamelCase_      ):






				"""simple docstring"""







				__a            ='trocr'
				__a            =['past_key_values']
				__a            ={
				    'num_attention_heads': 'decoder_attention_heads',
				    'hidden_size': 'd_model',
				    'num_hidden_layers': 'decoder_layers',
				}




				def __init__(  self     :					Optional[int]					,				__a     :					Any=5_02_65					,				__a     :					Optional[int]=10_24					,				__a     :					List[Any]=12					,				__a     :					str=16					,				__a     :					int=40_96					,				__a     :					Optional[Any]="gelu"					,				__a     :					Union[str, Any]=5_12					,				__a     :					Dict=0.1					,				__a     :					List[str]=0.0					,				__a     :					Union[str, Any]=0.0					,				__a     :					Any=2					,				__a     :					Union[str, Any]=0.02					,				__a     :					Any=0.0					,				__a     :					List[str]=True					,				__a     :					Optional[Any]=False					,				__a     :					Union[str, Any]=True					,				__a     :					Optional[Any]=True					,				__a     :					Any=1					,				__a     :					List[Any]=0					,				__a     :					Any=2					,				**__a     :					Optional[Any]					,				):
								_a								= vocab_size
								_a								= d_model
								_a								= decoder_layers
								_a								= decoder_attention_heads
								_a								= decoder_ffn_dim
								_a								= activation_function
								_a								= max_position_embeddings
								_a								= dropout
								_a								= attention_dropout
								_a								= activation_dropout
								_a								= init_std
								_a								= decoder_layerdrop
								_a								= use_cache
								_a								= scale_embedding
								_a								= use_learned_position_embeddings
								_a								= layernorm_embedding

								super().__init__(
								    pad_token_id=__a					,				bos_token_id=__a					,				eos_token_id=__a					,				decoder_start_token_id=__a					,				**__a					,				)
