
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__lowerCamelCase							=       logging.get_logger(__name__)

__lowerCamelCase							=       """▁"""

__lowerCamelCase							=       {"""vocab_file""": """sentencepiece.bpe.model"""}

__lowerCamelCase							=       {
    """vocab_file""": {
        """facebook/nllb-200-distilled-600M""": (
            """https://huggingface.co/facebook/nllb-200-distilled-600M/blob/main/sentencepiece.bpe.model"""
        ),
    }
}

__lowerCamelCase							=       {
    """facebook/nllb-200-distilled-600M""": 10_24,
}

# fmt: off
__lowerCamelCase							=       ["""ace_Arab""", """ace_Latn""", """acm_Arab""", """acq_Arab""", """aeb_Arab""", """afr_Latn""", """ajp_Arab""", """aka_Latn""", """amh_Ethi""", """apc_Arab""", """arb_Arab""", """ars_Arab""", """ary_Arab""", """arz_Arab""", """asm_Beng""", """ast_Latn""", """awa_Deva""", """ayr_Latn""", """azb_Arab""", """azj_Latn""", """bak_Cyrl""", """bam_Latn""", """ban_Latn""", """bel_Cyrl""", """bem_Latn""", """ben_Beng""", """bho_Deva""", """bjn_Arab""", """bjn_Latn""", """bod_Tibt""", """bos_Latn""", """bug_Latn""", """bul_Cyrl""", """cat_Latn""", """ceb_Latn""", """ces_Latn""", """cjk_Latn""", """ckb_Arab""", """crh_Latn""", """cym_Latn""", """dan_Latn""", """deu_Latn""", """dik_Latn""", """dyu_Latn""", """dzo_Tibt""", """ell_Grek""", """eng_Latn""", """epo_Latn""", """est_Latn""", """eus_Latn""", """ewe_Latn""", """fao_Latn""", """pes_Arab""", """fij_Latn""", """fin_Latn""", """fon_Latn""", """fra_Latn""", """fur_Latn""", """fuv_Latn""", """gla_Latn""", """gle_Latn""", """glg_Latn""", """grn_Latn""", """guj_Gujr""", """hat_Latn""", """hau_Latn""", """heb_Hebr""", """hin_Deva""", """hne_Deva""", """hrv_Latn""", """hun_Latn""", """hye_Armn""", """ibo_Latn""", """ilo_Latn""", """ind_Latn""", """isl_Latn""", """ita_Latn""", """jav_Latn""", """jpn_Jpan""", """kab_Latn""", """kac_Latn""", """kam_Latn""", """kan_Knda""", """kas_Arab""", """kas_Deva""", """kat_Geor""", """knc_Arab""", """knc_Latn""", """kaz_Cyrl""", """kbp_Latn""", """kea_Latn""", """khm_Khmr""", """kik_Latn""", """kin_Latn""", """kir_Cyrl""", """kmb_Latn""", """kon_Latn""", """kor_Hang""", """kmr_Latn""", """lao_Laoo""", """lvs_Latn""", """lij_Latn""", """lim_Latn""", """lin_Latn""", """lit_Latn""", """lmo_Latn""", """ltg_Latn""", """ltz_Latn""", """lua_Latn""", """lug_Latn""", """luo_Latn""", """lus_Latn""", """mag_Deva""", """mai_Deva""", """mal_Mlym""", """mar_Deva""", """min_Latn""", """mkd_Cyrl""", """plt_Latn""", """mlt_Latn""", """mni_Beng""", """khk_Cyrl""", """mos_Latn""", """mri_Latn""", """zsm_Latn""", """mya_Mymr""", """nld_Latn""", """nno_Latn""", """nob_Latn""", """npi_Deva""", """nso_Latn""", """nus_Latn""", """nya_Latn""", """oci_Latn""", """gaz_Latn""", """ory_Orya""", """pag_Latn""", """pan_Guru""", """pap_Latn""", """pol_Latn""", """por_Latn""", """prs_Arab""", """pbt_Arab""", """quy_Latn""", """ron_Latn""", """run_Latn""", """rus_Cyrl""", """sag_Latn""", """san_Deva""", """sat_Beng""", """scn_Latn""", """shn_Mymr""", """sin_Sinh""", """slk_Latn""", """slv_Latn""", """smo_Latn""", """sna_Latn""", """snd_Arab""", """som_Latn""", """sot_Latn""", """spa_Latn""", """als_Latn""", """srd_Latn""", """srp_Cyrl""", """ssw_Latn""", """sun_Latn""", """swe_Latn""", """swh_Latn""", """szl_Latn""", """tam_Taml""", """tat_Cyrl""", """tel_Telu""", """tgk_Cyrl""", """tgl_Latn""", """tha_Thai""", """tir_Ethi""", """taq_Latn""", """taq_Tfng""", """tpi_Latn""", """tsn_Latn""", """tso_Latn""", """tuk_Latn""", """tum_Latn""", """tur_Latn""", """twi_Latn""", """tzm_Tfng""", """uig_Arab""", """ukr_Cyrl""", """umb_Latn""", """urd_Arab""", """uzn_Latn""", """vec_Latn""", """vie_Latn""", """war_Latn""", """wol_Latn""", """xho_Latn""", """ydd_Hebr""", """yor_Latn""", """yue_Hant""", """zho_Hans""", """zho_Hant""", """zul_Latn"""]






class   UpperCAmelCase		(						A_ ):
   A__					:		Any						     =     VOCAB_FILES_NAMES
   A__					:		List[str]						     =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A__					:		List[str]						     =     PRETRAINED_VOCAB_FILES_MAP
   A__					:		int						     =     ["input_ids", "attention_mask"]

   A__					:		List[int]						     =     []
   A__					:		List[int]						     =     []



   def __init__(self		:					Optional[int]    ,						snake_case__		:					str    ,						snake_case__		:					int="<s>"    ,						snake_case__		:					int="</s>"    ,						snake_case__		:					Tuple="</s>"    ,						snake_case__		:					Optional[Any]="<s>"    ,						snake_case__		:					Tuple="<unk>"    ,						snake_case__		:					Tuple="<pad>"    ,						snake_case__		:					Union[str, Any]="<mask>"    ,						snake_case__		:					int=None    ,						snake_case__		:					Tuple=None    ,						snake_case__		:					Tuple=None    ,						snake_case__		:					Optional[Dict[str, Any]] = None    ,						snake_case__		:					Tuple=None    ,						snake_case__		:					Dict=False    ,						**snake_case__		:					Optional[int]    ,						)		->	List[Any]:


          '''simple docstring'''


          snake_case       :   str							=      AddedToken(snake_case__    ,						lstrip=snake_case__    ,						rstrip=snake_case__ ) if isinstance(snake_case__    ,						snake_case__ ) else mask_token

          snake_case       :   Any							=      {} if sp_model_kwargs is None else sp_model_kwargs
          snake_case       :   str							=      legacy_behaviour

          super().__init__(
              bos_token=snake_case__    ,						eos_token=snake_case__    ,						unk_token=snake_case__    ,						sep_token=snake_case__    ,						cls_token=snake_case__    ,						pad_token=snake_case__    ,						mask_token=snake_case__    ,						tokenizer_file=snake_case__    ,						src_lang=snake_case__    ,						tgt_lang=snake_case__    ,						additional_special_tokens=snake_case__    ,						sp_model_kwargs=self.sp_model_kwargs    ,						legacy_behaviour=snake_case__    ,						**snake_case__    ,						)

          snake_case       :   List[str]							=      spm.SentencePieceProcessor(**self.sp_model_kwargs )
          self.sp_model.Load(str(snake_case__ ) )
          snake_case       :   str							=      vocab_file
          # Original fairseq vocab and spm vocab must be "aligned":
          # Vocab    |    0    |    1    |   2    |    3    |  4   |  5   |  6   |   7  |   8  |  9
          # -------- | ------- | ------- | ------ | ------- | ---- | ---- | ---- | ---- | ---- | ----
          # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | 'an' | '▁n' | '▁m' | '▁t' | '▁k' | '▁a'
          # spm      | '<unk>' | '<s>'   | '</s>' | 'an'    | '▁n' | '▁m' | '▁t' | '▁k' | '▁a' | '▁s'

          # Mimic fairseq token-to-id alignment for the first 4 token
          snake_case       :   Optional[int]							=      {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

          # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
          snake_case       :   Union[str, Any]							=      1

          snake_case       :   Optional[Any]							=      len(self.sp_model )
          snake_case       :   List[Any]							=      {
              code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(snake_case__ )
          }
          snake_case       :   str							=      {v: k for k, v in self.lang_code_to_id.items()}
          snake_case       :   Optional[int]							=      len(self.sp_model ) + len(self.lang_code_to_id ) + self.fairseq_offset

          self.fairseq_tokens_to_ids.update(self.lang_code_to_id )
          snake_case       :   str							=      {v: k for k, v in self.fairseq_tokens_to_ids.items()}
          snake_case       :   Optional[int]							=      list(self.lang_code_to_id.keys() )

          if additional_special_tokens is not None:
                 # Only add those special tokens if they are not already there.
                 self._additional_special_tokens.extend(
                     [t for t in additional_special_tokens if t not in self._additional_special_tokens] )

          snake_case       :   Optional[Any]							=      src_lang if src_lang is not None else "eng_Latn"
          snake_case       :   Union[str, Any]							=      self.lang_code_to_id[self._src_lang]
          snake_case       :   List[Any]							=      tgt_lang
          self.set_src_lang_special_tokens(self._src_lang )



   def __getstate__(self		:					Optional[int] )		->	Dict:


          '''simple docstring'''


          snake_case       :   str							=      self.__dict__.copy()
          snake_case       :   List[Any]							=      None
          snake_case       :   Tuple							=      self.sp_model.serialized_model_proto()
          return state



   def __setstate__(self		:					Optional[Any]    ,						snake_case__		:					Tuple )		->	Any:


          '''simple docstring'''


          snake_case       :   List[Any]							=      d

          # for backward compatibility
          if not hasattr(self    ,						"sp_model_kwargs" ):
                 snake_case       :   str							=      {}

          snake_case       :   List[Any]							=      spm.SentencePieceProcessor(**self.sp_model_kwargs )
          self.sp_model.LoadFromSerializedProto(self.sp_model_proto )



   @property
   def 				_SCREAMING_SNAKE_CASE   (self		:					Optional[Any] )		->	List[Any]:


          '''simple docstring'''


          return len(self.sp_model ) + len(self.lang_code_to_id ) + self.fairseq_offset + 1  # Plus 1 for the mask token



   @property
   def 				_SCREAMING_SNAKE_CASE   (self		:					Optional[int] )		->	str:


          '''simple docstring'''


          return self._src_lang



   @src_lang.setter
   def 				_SCREAMING_SNAKE_CASE   (self		:					List[str]    ,						snake_case__		:					str )		->	None:


          '''simple docstring'''


          snake_case       :   int							=      new_src_lang
          self.set_src_lang_special_tokens(self._src_lang )



   def 				_SCREAMING_SNAKE_CASE   (self		:					Any    ,						snake_case__		:					List[int]    ,						snake_case__		:					Optional[List[int]] = None    ,						snake_case__		:					bool = False )		->	List[int]:


          '''simple docstring'''


          if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=snake_case__    ,						token_ids_a=snake_case__    ,						already_has_special_tokens=snake_case__ )

          snake_case       :   Tuple							=      [1] * len(self.prefix_tokens )
          snake_case       :   Dict							=      [1] * len(self.suffix_tokens )
          if token_ids_a is None:
                 return prefix_ones + ([0] * len(snake_case__ )) + suffix_ones
          return prefix_ones + ([0] * len(snake_case__ )) + ([0] * len(snake_case__ )) + suffix_ones



   def 				_SCREAMING_SNAKE_CASE   (self		:					List[Any]    ,						snake_case__		:					List[int]    ,						snake_case__		:					Optional[List[int]] = None )		->	List[int]:


          '''simple docstring'''


          if token_ids_a is None:
                 return self.prefix_tokens + token_ids_a + self.suffix_tokens
          # We don't expect to process pairs, but leave the pair logic for API consistency
          return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



   def 				_SCREAMING_SNAKE_CASE   (self		:					int    ,						snake_case__		:					List[int]    ,						snake_case__		:					Optional[List[int]] = None )		->	List[int]:


          '''simple docstring'''


          snake_case       :   List[str]							=      [self.sep_token_id]
          snake_case       :   Union[str, Any]							=      [self.cls_token_id]

          if token_ids_a is None:
                 return len(cls + token_ids_a + sep ) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]



   def 				_SCREAMING_SNAKE_CASE   (self		:					Any    ,						snake_case__		:					int    ,						snake_case__		:					str    ,						snake_case__		:					Optional[str]    ,						snake_case__		:					Optional[str]    ,						**snake_case__		:					Tuple )		->	List[str]:


          '''simple docstring'''


          if src_lang is None or tgt_lang is None:
                 raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model" )
          snake_case       :   Optional[Any]							=      src_lang
          snake_case       :   Union[str, Any]							=      self(snake_case__    ,						add_special_tokens=snake_case__    ,						return_tensors=snake_case__    ,						**snake_case__ )
          snake_case       :   Optional[Any]							=      self.convert_tokens_to_ids(snake_case__ )
          snake_case       :   Dict							=      tgt_lang_id
          return inputs



   def 				_SCREAMING_SNAKE_CASE   (self		:					Any )		->	int:


          '''simple docstring'''


          snake_case       :   List[Any]							=      {self.convert_ids_to_tokens(snake_case__ ): i for i in range(self.vocab_size )}
          vocab.update(self.added_tokens_encoder )
          return vocab



   def 				_SCREAMING_SNAKE_CASE   (self		:					List[str]    ,						snake_case__		:					str )		->	List[str]:


          '''simple docstring'''


          return self.sp_model.encode(snake_case__    ,						out_type=snake_case__ )



   def 				_SCREAMING_SNAKE_CASE   (self		:					Any    ,						snake_case__		:					Any )		->	Optional[int]:


          '''simple docstring'''


          if token in self.fairseq_tokens_to_ids:
                 return self.fairseq_tokens_to_ids[token]
          snake_case       :   int							=      self.sp_model.PieceToId(snake_case__ )

          # Need to return unknown token if the SP model returned 0
          return spm_id + self.fairseq_offset if spm_id else self.unk_token_id



   def 				_SCREAMING_SNAKE_CASE   (self		:					List[str]    ,						snake_case__		:					Tuple )		->	str:


          '''simple docstring'''


          if index in self.fairseq_ids_to_tokens:
                 return self.fairseq_ids_to_tokens[index]
          return self.sp_model.IdToPiece(index - self.fairseq_offset )



   def 				_SCREAMING_SNAKE_CASE   (self		:					Dict    ,						snake_case__		:					Optional[Any] )		->	List[str]:


          '''simple docstring'''


          snake_case       :   Optional[int]							=      "".join(snake_case__ ).replace(snake_case__    ,						" " ).strip()
          return out_string



   def 				_SCREAMING_SNAKE_CASE   (self		:					Optional[int]    ,						snake_case__		:					str    ,						snake_case__		:					Optional[str] = None )		->	Tuple[str]:


          '''simple docstring'''


          if not os.path.isdir(snake_case__ ):
                 logger.error(f"""Vocabulary path ({save_directory}) should be a directory""" )
                 return
          snake_case       :   Optional[int]							=      os.path.join(
              snake_case__    ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )

          if os.path.abspath(self.vocab_file ) != os.path.abspath(snake_case__ ) and os.path.isfile(self.vocab_file ):
                 copyfile(self.vocab_file    ,						snake_case__ )
          elif not os.path.isfile(self.vocab_file ):
                 with open(snake_case__    ,						"wb" ) as fi:
                        snake_case       :   Union[str, Any]							=      self.sp_model.serialized_model_proto()
                        fi.write(snake_case__ )

          return (out_vocab_file,)



   def 				_SCREAMING_SNAKE_CASE   (self		:					List[Any]    ,						snake_case__		:					List[str]    ,						snake_case__		:					str = "eng_Latn"    ,						snake_case__		:					Optional[List[str]] = None    ,						snake_case__		:					str = "fra_Latn"    ,						**snake_case__		:					int    ,						)		->	BatchEncoding:


          '''simple docstring'''


          snake_case       :   Optional[int]							=      src_lang
          snake_case       :   Optional[int]							=      tgt_lang
          return super().prepare_seqaseq_batch(snake_case__    ,						snake_case__    ,						**snake_case__ )



   def 				_SCREAMING_SNAKE_CASE   (self		:					Optional[int] )		->	Optional[int]:


          '''simple docstring'''


          return self.set_src_lang_special_tokens(self.src_lang )



   def 				_SCREAMING_SNAKE_CASE   (self		:					str )		->	List[Any]:


          '''simple docstring'''


          return self.set_tgt_lang_special_tokens(self.tgt_lang )



   def 				_SCREAMING_SNAKE_CASE   (self		:					Optional[Any]    ,						snake_case__		:					str )		->	None:


          '''simple docstring'''


          snake_case       :   str							=      self.lang_code_to_id[src_lang]
          if self.legacy_behaviour:
                 snake_case       :   Union[str, Any]							=      []
                 snake_case       :   Optional[int]							=      [self.eos_token_id, self.cur_lang_code]
          else:
                 snake_case       :   Dict							=      [self.cur_lang_code]
                 snake_case       :   Any							=      [self.eos_token_id]



   def 				_SCREAMING_SNAKE_CASE   (self		:					List[str]    ,						snake_case__		:					str )		->	None:


          '''simple docstring'''


          snake_case       :   int							=      self.lang_code_to_id[lang]
          if self.legacy_behaviour:
                 snake_case       :   int							=      []
                 snake_case       :   List[Any]							=      [self.eos_token_id, self.cur_lang_code]
          else:
                 snake_case       :   List[Any]							=      [self.cur_lang_code]
                 snake_case       :   Optional[Any]							=      [self.eos_token_id]









"""simple docstring"""





snake_case__						:   str			 = [
    999,
    800,
    799,
    600,
    599,
    500,
    400,
    399,
    377,
    355,
    333,
    311,
    288,
    266,
    244,
    222,
    200,
    199,
    177,
    155,
    133,
    111,
    88,
    66,
    44,
    22,
    0,
]

snake_case__						:   Optional[Any]			 = [
    999,
    976,
    952,
    928,
    905,
    882,
    858,
    857,
    810,
    762,
    715,
    714,
    572,
    429,
    428,
    286,
    285,
    238,
    190,
    143,
    142,
    118,
    95,
    71,
    47,
    24,
    0,
]

snake_case__						:   Any			 = [
    999,
    988,
    977,
    966,
    955,
    944,
    933,
    922,
    911,
    900,
    899,
    879,
    859,
    840,
    820,
    800,
    799,
    766,
    733,
    700,
    699,
    650,
    600,
    599,
    500,
    499,
    400,
    399,
    350,
    300,
    299,
    266,
    233,
    200,
    199,
    179,
    159,
    140,
    120,
    100,
    99,
    88,
    77,
    66,
    55,
    44,
    33,
    22,
    11,
    0,
]

snake_case__						:   Optional[Any]			 = [
    999,
    995,
    992,
    989,
    985,
    981,
    978,
    975,
    971,
    967,
    964,
    961,
    957,
    956,
    951,
    947,
    942,
    937,
    933,
    928,
    923,
    919,
    914,
    913,
    908,
    903,
    897,
    892,
    887,
    881,
    876,
    871,
    870,
    864,
    858,
    852,
    846,
    840,
    834,
    828,
    827,
    820,
    813,
    806,
    799,
    792,
    785,
    784,
    777,
    770,
    763,
    756,
    749,
    742,
    741,
    733,
    724,
    716,
    707,
    699,
    698,
    688,
    677,
    666,
    656,
    655,
    645,
    634,
    623,
    613,
    612,
    598,
    584,
    570,
    569,
    555,
    541,
    527,
    526,
    505,
    484,
    483,
    462,
    440,
    439,
    396,
    395,
    352,
    351,
    308,
    307,
    264,
    263,
    220,
    219,
    176,
    132,
    88,
    44,
    0,
]

snake_case__						:   int			 = [
    999,
    997,
    995,
    992,
    990,
    988,
    986,
    984,
    981,
    979,
    977,
    975,
    972,
    970,
    968,
    966,
    964,
    961,
    959,
    957,
    956,
    954,
    951,
    949,
    946,
    944,
    941,
    939,
    936,
    934,
    931,
    929,
    926,
    924,
    921,
    919,
    916,
    914,
    913,
    910,
    907,
    905,
    902,
    899,
    896,
    893,
    891,
    888,
    885,
    882,
    879,
    877,
    874,
    871,
    870,
    867,
    864,
    861,
    858,
    855,
    852,
    849,
    846,
    843,
    840,
    837,
    834,
    831,
    828,
    827,
    824,
    821,
    817,
    814,
    811,
    808,
    804,
    801,
    798,
    795,
    791,
    788,
    785,
    784,
    780,
    777,
    774,
    770,
    766,
    763,
    760,
    756,
    752,
    749,
    746,
    742,
    741,
    737,
    733,
    730,
    726,
    722,
    718,
    714,
    710,
    707,
    703,
    699,
    698,
    694,
    690,
    685,
    681,
    677,
    673,
    669,
    664,
    660,
    656,
    655,
    650,
    646,
    641,
    636,
    632,
    627,
    622,
    618,
    613,
    612,
    607,
    602,
    596,
    591,
    586,
    580,
    575,
    570,
    569,
    563,
    557,
    551,
    545,
    539,
    533,
    527,
    526,
    519,
    512,
    505,
    498,
    491,
    484,
    483,
    474,
    466,
    457,
    449,
    440,
    439,
    428,
    418,
    407,
    396,
    395,
    381,
    366,
    352,
    351,
    330,
    308,
    307,
    286,
    264,
    263,
    242,
    220,
    219,
    176,
    175,
    132,
    131,
    88,
    44,
    0,
]

snake_case__						:   Union[str, Any]			 = [
    999,
    991,
    982,
    974,
    966,
    958,
    950,
    941,
    933,
    925,
    916,
    908,
    900,
    899,
    874,
    850,
    825,
    800,
    799,
    700,
    600,
    500,
    400,
    300,
    200,
    100,
    0,
]

snake_case__						:   List[Any]			 = [
    999,
    992,
    985,
    978,
    971,
    964,
    957,
    949,
    942,
    935,
    928,
    921,
    914,
    907,
    900,
    899,
    879,
    859,
    840,
    820,
    800,
    799,
    766,
    733,
    700,
    699,
    650,
    600,
    599,
    500,
    499,
    400,
    399,
    300,
    299,
    200,
    199,
    100,
    99,
    0,
]

snake_case__						:   Optional[int]			 = [
    999,
    996,
    992,
    989,
    985,
    982,
    979,
    975,
    972,
    968,
    965,
    961,
    958,
    955,
    951,
    948,
    944,
    941,
    938,
    934,
    931,
    927,
    924,
    920,
    917,
    914,
    910,
    907,
    903,
    900,
    899,
    891,
    884,
    876,
    869,
    861,
    853,
    846,
    838,
    830,
    823,
    815,
    808,
    800,
    799,
    788,
    777,
    766,
    755,
    744,
    733,
    722,
    711,
    700,
    699,
    688,
    677,
    666,
    655,
    644,
    633,
    622,
    611,
    600,
    599,
    585,
    571,
    557,
    542,
    528,
    514,
    500,
    499,
    485,
    471,
    457,
    442,
    428,
    414,
    400,
    399,
    379,
    359,
    340,
    320,
    300,
    299,
    279,
    259,
    240,
    220,
    200,
    199,
    166,
    133,
    100,
    99,
    66,
    33,
    0,
]


