





'''simple docstring'''






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
     from .tokenization_mbartaa import MBartaaTokenizer
else:
     snake_case_     			=						None


snake_case_     			=						logging.get_logger(__name__)

snake_case_     			=						{"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

snake_case_     			=						{
    """vocab_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json"""
        ),
    },
}

snake_case_     			=						{
    """facebook/mbart-large-50-one-to-many-mmt""": 10_24,
}

# fmt: off
snake_case_     			=						["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN""", """af_ZA""", """az_AZ""", """bn_IN""", """fa_IR""", """he_IL""", """hr_HR""", """id_ID""", """ka_GE""", """km_KH""", """mk_MK""", """ml_IN""", """mn_MN""", """mr_IN""", """pl_PL""", """ps_AF""", """pt_XX""", """sv_SE""", """sw_KE""", """ta_IN""", """te_IN""", """th_TH""", """tl_XX""", """uk_UA""", """ur_PK""", """xh_ZA""", """gl_ES""", """sl_SI"""]


class 	_lowercase    (				_snake_case					):
   _UpperCamelCase									= VOCAB_FILES_NAMES
   _UpperCamelCase									= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _UpperCamelCase									= PRETRAINED_VOCAB_FILES_MAP
   _UpperCamelCase									= ["""input_ids""", """attention_mask"""]
   _UpperCamelCase									= MBartaaTokenizer

   _UpperCamelCase									= []
   _UpperCamelCase									= []


   def __init__(			self     , _UpperCAmelCase=None     , _UpperCAmelCase=None     , _UpperCAmelCase=None     , _UpperCAmelCase=None     , _UpperCAmelCase="</s>"     , _UpperCAmelCase="</s>"     , _UpperCAmelCase="<s>"     , _UpperCAmelCase="<unk>"     , _UpperCAmelCase="<pad>"     , _UpperCAmelCase="<mask>"     , **_UpperCAmelCase     , ):
       # Mask token behave like a normal word, i.e. include the space before it
       A							:							str				=    AddedToken(lowercase_     , lstrip=lowercase_     , rstrip=lowercase_       ) if isinstance(lowercase_     , lowercase_       ) else mask_token

       A							:							Optional[int]				=    kwargs.get('''additional_special_tokens'''     , []       )
       kwargs["additional_special_tokens"] += [
           code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
       ]

       super().__init__(
           lowercase_     , src_lang=lowercase_     , tgt_lang=lowercase_     , tokenizer_file=lowercase_     , eos_token=lowercase_     , sep_token=lowercase_     , cls_token=lowercase_     , unk_token=lowercase_     , pad_token=lowercase_     , mask_token=lowercase_     , **lowercase_     , )

       A							:							List[Any]				=    vocab_file
       A							:							Union[str, Any]				=    False if not self.vocab_file else True

       A							:							List[Any]				=    {
           lang_code: self.convert_tokens_to_ids(lowercase_       ) for lang_code in FAIRSEQ_LANGUAGE_CODES
       }

       A							:							Any				=    src_lang if src_lang is not None else 'en_XX'
       A							:							Any				=    tgt_lang
       A							:							List[Any]				=    self.lang_code_to_id[self._src_lang]
       self.set_src_lang_special_tokens(self._src_lang       )


   @property
   def        snake_case					(			self       ):
       return self._src_lang


   @src_lang.setter
   def        snake_case					(			self     , _UpperCAmelCase       ):
       A							:							Optional[int]				=    new_src_lang
       self.set_src_lang_special_tokens(self._src_lang       )


   def        snake_case					(			self     , _UpperCAmelCase     , _UpperCAmelCase = None       ):
       if token_ids_a is None:
           return self.prefix_tokens + token_ids_a + self.suffix_tokens
       # We don't expect to process pairs, but leave the pair logic for API consistency
       return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens


   def        snake_case					(			self     , _UpperCAmelCase     , _UpperCAmelCase = "en_XX"     , _UpperCAmelCase = None     , _UpperCAmelCase = "ro_RO"     , **_UpperCAmelCase     , ):
       A							:							Optional[int]				=    src_lang
       A							:							List[str]				=    tgt_lang
       return super().prepare_seqaseq_batch(lowercase_     , lowercase_     , **lowercase_       )


   def        snake_case					(			self       ):
       return self.set_src_lang_special_tokens(self.src_lang       )


   def        snake_case					(			self       ):
       return self.set_tgt_lang_special_tokens(self.tgt_lang       )


   def        snake_case					(			self     , _UpperCAmelCase       ):
       A							:							Dict				=    self.convert_tokens_to_ids(lowercase_       )
       A							:							str				=    [self.cur_lang_code_id]
       A							:							Dict				=    [self.eos_token_id]

       A							:							int				=    self.convert_ids_to_tokens(self.prefix_tokens       )
       A							:							Optional[int]				=    self.convert_ids_to_tokens(self.suffix_tokens       )

       A							:							Optional[int]				=    processors.TemplateProcessing(
           single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str     , pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str     , special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str     , self.prefix_tokens + self.suffix_tokens       )       )     , )


   def        snake_case					(			self     , _UpperCAmelCase       ):
       A							:							Optional[Any]				=    self.convert_tokens_to_ids(lowercase_       )
       A							:							List[str]				=    [self.cur_lang_code_id]
       A							:							List[Any]				=    [self.eos_token_id]

       A							:							Optional[Any]				=    self.convert_ids_to_tokens(self.prefix_tokens       )
       A							:							Union[str, Any]				=    self.convert_ids_to_tokens(self.suffix_tokens       )

       A							:							str				=    processors.TemplateProcessing(
           single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str     , pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str     , special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str     , self.prefix_tokens + self.suffix_tokens       )       )     , )


   def        snake_case					(			self     , _UpperCAmelCase     , _UpperCAmelCase     , _UpperCAmelCase     , _UpperCAmelCase     , **_UpperCAmelCase       ):
       if src_lang is None or tgt_lang is None:
           raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''       )
       A							:							Optional[Any]				=    src_lang
       A							:							List[Any]				=    self(lowercase_     , add_special_tokens=lowercase_     , return_tensors=lowercase_     , **lowercase_       )
       A							:							Dict				=    self.convert_tokens_to_ids(lowercase_       )
       A							:							Any				=    tgt_lang_id
       return inputs


   def        snake_case					(			self     , _UpperCAmelCase     , _UpperCAmelCase = None       ):
       if not self.can_save_slow_tokenizer:
           raise ValueError(
               '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
               '''tokenizer.'''       )

       if not os.path.isdir(lowercase_       ):
           logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''       )
           return
       A							:							Optional[int]				=    os.path.join(
           lowercase_     , (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )

       if os.path.abspath(self.vocab_file       ) != os.path.abspath(lowercase_       ):
           copyfile(self.vocab_file     , lowercase_       )

       return (out_vocab_file,)



'''simple docstring'''

from __future__ import annotations




def  UpperCamelCase(	UpperCAmelCase_    ,			UpperCAmelCase_    ,			UpperCAmelCase_      ):
 assert n > 1 and a > 0 and greatest_common_divisor(UpperCAmelCase_    ,			UpperCAmelCase_      ) == 1
 ((UpperCAmelCase)   ,					(UpperCAmelCase)   ,					(UpperCAmelCase))			:      Dict		   =					extended_gcd(UpperCAmelCase_    ,			UpperCAmelCase_      )  # Implemented below
 UpperCAmelCase			:      int		   =					(b * s) % n
 return x




def  UpperCamelCase(	UpperCAmelCase_    ,			UpperCAmelCase_      ):
 ((UpperCAmelCase)   ,					(UpperCAmelCase))			:      int		   =					extended_euclid(UpperCAmelCase_    ,			UpperCAmelCase_      )  # Implemented below
 if b < 0:
  UpperCAmelCase			:      Optional[int]		   =					(b % n + n) % n
 return b




def  UpperCamelCase(	UpperCAmelCase_    ,			UpperCAmelCase_    ,			UpperCAmelCase_      ):
 UpperCAmelCase			:      Tuple		   =					invert_modulo(UpperCAmelCase_    ,			UpperCAmelCase_      )
 UpperCAmelCase			:      List[Any]		   =					(b * s) % n
 return x




def  UpperCamelCase(	UpperCAmelCase_    ,			UpperCAmelCase_      ):
 assert a >= 0 and b >= 0

 if b == 0:
  UpperCAmelCase   ,					UpperCAmelCase   ,					UpperCAmelCase			:      Dict		   =					a, 1, 0
 else:
  ((UpperCAmelCase)   ,					(UpperCAmelCase)   ,					(UpperCAmelCase))			:      Optional[int]		   =					extended_gcd(UpperCAmelCase_    ,			a % b      )
  UpperCAmelCase			:      Dict		   =					q
  UpperCAmelCase			:      int		   =					p - q * (a // b)

 assert a % d == 0 and b % d == 0
 assert d == a * x + b * y

 return (d, x, y)




def  UpperCamelCase(	UpperCAmelCase_    ,			UpperCAmelCase_      ):
 if b == 0:
  return (1, 0)
 ((UpperCAmelCase)   ,					(UpperCAmelCase))			:      Tuple		   =					extended_euclid(UpperCAmelCase_    ,			a % b      )
 UpperCAmelCase			:      Tuple		   =					a // b
 return (y, x - k * y)




def  UpperCamelCase(	UpperCAmelCase_    ,			UpperCAmelCase_      ):
 if a < b:
  UpperCAmelCase   ,					UpperCAmelCase			:      Union[str, Any]		   =					b, a

 while a % b != 0:
  UpperCAmelCase   ,					UpperCAmelCase			:      int		   =					b, a % b

 return b


if __name__ == "__main__":
 from doctest import testmod

 testmod(name="modular_division", verbose=True)
 testmod(name="modular_division2", verbose=True)
 testmod(name="invert_modulo", verbose=True)
 testmod(name="extended_gcd", verbose=True)
 testmod(name="extended_euclid", verbose=True)
 testmod(name="greatest_common_divisor", verbose=True)




