"""simple docstring"""


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
    from .tokenization_nllb import NllbTokenizer
else:
    __UpperCamelCase									=			None


__UpperCamelCase									=			logging.get_logger(__name__)


__UpperCamelCase									=			{'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

__UpperCamelCase									=			{
    '''vocab_file''': {
        '''facebook/nllb-200-distilled-600M''': (
            '''https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/nllb-200-distilled-600M''': (
            '''https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json'''
        ),
    },
}

__UpperCamelCase									=			{
    '''facebook/nllb-large-en-ro''': 1024,
    '''facebook/nllb-200-distilled-600M''': 1024,
}

# fmt: off
__UpperCamelCase									=			['''ace_Arab''', '''ace_Latn''', '''acm_Arab''', '''acq_Arab''', '''aeb_Arab''', '''afr_Latn''', '''ajp_Arab''', '''aka_Latn''', '''amh_Ethi''', '''apc_Arab''', '''arb_Arab''', '''ars_Arab''', '''ary_Arab''', '''arz_Arab''', '''asm_Beng''', '''ast_Latn''', '''awa_Deva''', '''ayr_Latn''', '''azb_Arab''', '''azj_Latn''', '''bak_Cyrl''', '''bam_Latn''', '''ban_Latn''', '''bel_Cyrl''', '''bem_Latn''', '''ben_Beng''', '''bho_Deva''', '''bjn_Arab''', '''bjn_Latn''', '''bod_Tibt''', '''bos_Latn''', '''bug_Latn''', '''bul_Cyrl''', '''cat_Latn''', '''ceb_Latn''', '''ces_Latn''', '''cjk_Latn''', '''ckb_Arab''', '''crh_Latn''', '''cym_Latn''', '''dan_Latn''', '''deu_Latn''', '''dik_Latn''', '''dyu_Latn''', '''dzo_Tibt''', '''ell_Grek''', '''eng_Latn''', '''epo_Latn''', '''est_Latn''', '''eus_Latn''', '''ewe_Latn''', '''fao_Latn''', '''pes_Arab''', '''fij_Latn''', '''fin_Latn''', '''fon_Latn''', '''fra_Latn''', '''fur_Latn''', '''fuv_Latn''', '''gla_Latn''', '''gle_Latn''', '''glg_Latn''', '''grn_Latn''', '''guj_Gujr''', '''hat_Latn''', '''hau_Latn''', '''heb_Hebr''', '''hin_Deva''', '''hne_Deva''', '''hrv_Latn''', '''hun_Latn''', '''hye_Armn''', '''ibo_Latn''', '''ilo_Latn''', '''ind_Latn''', '''isl_Latn''', '''ita_Latn''', '''jav_Latn''', '''jpn_Jpan''', '''kab_Latn''', '''kac_Latn''', '''kam_Latn''', '''kan_Knda''', '''kas_Arab''', '''kas_Deva''', '''kat_Geor''', '''knc_Arab''', '''knc_Latn''', '''kaz_Cyrl''', '''kbp_Latn''', '''kea_Latn''', '''khm_Khmr''', '''kik_Latn''', '''kin_Latn''', '''kir_Cyrl''', '''kmb_Latn''', '''kon_Latn''', '''kor_Hang''', '''kmr_Latn''', '''lao_Laoo''', '''lvs_Latn''', '''lij_Latn''', '''lim_Latn''', '''lin_Latn''', '''lit_Latn''', '''lmo_Latn''', '''ltg_Latn''', '''ltz_Latn''', '''lua_Latn''', '''lug_Latn''', '''luo_Latn''', '''lus_Latn''', '''mag_Deva''', '''mai_Deva''', '''mal_Mlym''', '''mar_Deva''', '''min_Latn''', '''mkd_Cyrl''', '''plt_Latn''', '''mlt_Latn''', '''mni_Beng''', '''khk_Cyrl''', '''mos_Latn''', '''mri_Latn''', '''zsm_Latn''', '''mya_Mymr''', '''nld_Latn''', '''nno_Latn''', '''nob_Latn''', '''npi_Deva''', '''nso_Latn''', '''nus_Latn''', '''nya_Latn''', '''oci_Latn''', '''gaz_Latn''', '''ory_Orya''', '''pag_Latn''', '''pan_Guru''', '''pap_Latn''', '''pol_Latn''', '''por_Latn''', '''prs_Arab''', '''pbt_Arab''', '''quy_Latn''', '''ron_Latn''', '''run_Latn''', '''rus_Cyrl''', '''sag_Latn''', '''san_Deva''', '''sat_Beng''', '''scn_Latn''', '''shn_Mymr''', '''sin_Sinh''', '''slk_Latn''', '''slv_Latn''', '''smo_Latn''', '''sna_Latn''', '''snd_Arab''', '''som_Latn''', '''sot_Latn''', '''spa_Latn''', '''als_Latn''', '''srd_Latn''', '''srp_Cyrl''', '''ssw_Latn''', '''sun_Latn''', '''swe_Latn''', '''swh_Latn''', '''szl_Latn''', '''tam_Taml''', '''tat_Cyrl''', '''tel_Telu''', '''tgk_Cyrl''', '''tgl_Latn''', '''tha_Thai''', '''tir_Ethi''', '''taq_Latn''', '''taq_Tfng''', '''tpi_Latn''', '''tsn_Latn''', '''tso_Latn''', '''tuk_Latn''', '''tum_Latn''', '''tur_Latn''', '''twi_Latn''', '''tzm_Tfng''', '''uig_Arab''', '''ukr_Cyrl''', '''umb_Latn''', '''urd_Arab''', '''uzn_Latn''', '''vec_Latn''', '''vie_Latn''', '''war_Latn''', '''wol_Latn''', '''xho_Latn''', '''ydd_Hebr''', '''yor_Latn''', '''yue_Hant''', '''zho_Hans''', '''zho_Hant''', '''zul_Latn''']




class  UpperCamelCase					(				lowerCAmelCase__   ):
       SCREAMING_SNAKE_CASE_			=      VOCAB_FILES_NAMES
       SCREAMING_SNAKE_CASE_			=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       SCREAMING_SNAKE_CASE_			=      PRETRAINED_VOCAB_FILES_MAP
       SCREAMING_SNAKE_CASE_			=      ["input_ids", "attention_mask"]
       SCREAMING_SNAKE_CASE_			=      NllbTokenizer

       SCREAMING_SNAKE_CASE_			=      []
       SCREAMING_SNAKE_CASE_			=      []







       def __init__( self,							lowerCAmelCase__=None,							lowerCAmelCase__=None,							lowerCAmelCase__="<s>",							lowerCAmelCase__="</s>",							lowerCAmelCase__="</s>",							lowerCAmelCase__="<s>",							lowerCAmelCase__="<unk>",							lowerCAmelCase__="<pad>",							lowerCAmelCase__="<mask>",							lowerCAmelCase__=None,							lowerCAmelCase__=None,							lowerCAmelCase__=None,							lowerCAmelCase__=False,							**lowerCAmelCase__,							)      ->      List[str]:
            # Mask token behave like a normal word, i.e. include the space before it
            snake_case_					       =					AddedToken(lowerCAmelCase__,							lstrip=lowerCAmelCase__,							rstrip=lowerCAmelCase__) if isinstance(lowerCAmelCase__,							lowerCAmelCase__) else mask_token
            snake_case_					       =					legacy_behaviour
            super().__init__(
                vocab_file=lowerCAmelCase__,							tokenizer_file=lowerCAmelCase__,							bos_token=lowerCAmelCase__,							eos_token=lowerCAmelCase__,							sep_token=lowerCAmelCase__,							cls_token=lowerCAmelCase__,							unk_token=lowerCAmelCase__,							pad_token=lowerCAmelCase__,							mask_token=lowerCAmelCase__,							src_lang=lowerCAmelCase__,							tgt_lang=lowerCAmelCase__,							additional_special_tokens=lowerCAmelCase__,							legacy_behaviour=lowerCAmelCase__,							**lowerCAmelCase__,							)

            snake_case_					       =					vocab_file
            snake_case_					       =					False if not self.vocab_file else True

            snake_case_					       =					FAIRSEQ_LANGUAGE_CODES.copy()

            if additional_special_tokens is not None:
                 # Only add those special tokens if they are not already there.
                 _additional_special_tokens.extend(
                     [t for t in additional_special_tokens if t not in _additional_special_tokens])

            self.add_special_tokens({'additional_special_tokens': _additional_special_tokens})
            snake_case_					       =					{
                lang_code: self.convert_tokens_to_ids(lowerCAmelCase__) for lang_code in FAIRSEQ_LANGUAGE_CODES
            }

            snake_case_					       =					src_lang if src_lang is not None else 'eng_Latn'
            snake_case_					       =					self.convert_tokens_to_ids(self._src_lang)
            snake_case_					       =					tgt_lang
            self.set_src_lang_special_tokens(self._src_lang)







       @property
       def 		a_						( self)      ->      str:
            return self._src_lang







       @src_lang.setter
       def 		a_						( self,							lowerCAmelCase__)      ->      None:
            snake_case_					       =					new_src_lang
            self.set_src_lang_special_tokens(self._src_lang)







       def 		a_						( self,							lowerCAmelCase__,							lowerCAmelCase__ = None)      ->      List[int]:
            if token_ids_a is None:
                 return self.prefix_tokens + token_ids_a + self.suffix_tokens
            # We don't expect to process pairs, but leave the pair logic for API consistency
            return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







       def 		a_						( self,							lowerCAmelCase__,							lowerCAmelCase__ = None)      ->      List[int]:
            snake_case_					       =					[self.sep_token_id]
            snake_case_					       =					[self.cls_token_id]

            if token_ids_a is None:
                 return len(cls + token_ids_a + sep) * [0]
            return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]







       def 		a_						( self,							lowerCAmelCase__,							lowerCAmelCase__,							lowerCAmelCase__,							lowerCAmelCase__,							**lowerCAmelCase__)      ->      str:
            if src_lang is None or tgt_lang is None:
                 raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model')
            snake_case_					       =					src_lang
            snake_case_					       =					self(lowerCAmelCase__,							add_special_tokens=lowerCAmelCase__,							return_tensors=lowerCAmelCase__,							**lowerCAmelCase__)
            snake_case_					       =					self.convert_tokens_to_ids(lowerCAmelCase__)
            snake_case_					       =					tgt_lang_id
            return inputs







       def 		a_						( self,							lowerCAmelCase__,							lowerCAmelCase__ = "eng_Latn",							lowerCAmelCase__ = None,							lowerCAmelCase__ = "fra_Latn",							**lowerCAmelCase__,							)      ->      BatchEncoding:
            snake_case_					       =					src_lang
            snake_case_					       =					tgt_lang
            return super().prepare_seqaseq_batch(lowerCAmelCase__,							lowerCAmelCase__,							**lowerCAmelCase__)







       def 		a_						( self)      ->      List[Any]:
            return self.set_src_lang_special_tokens(self.src_lang)







       def 		a_						( self)      ->      Tuple:
            return self.set_tgt_lang_special_tokens(self.tgt_lang)







       def 		a_						( self,							lowerCAmelCase__)      ->      None:
            snake_case_					       =					self.convert_tokens_to_ids(lowerCAmelCase__)

            if self.legacy_behaviour:
                 snake_case_					       =					[]
                 snake_case_					       =					[self.eos_token_id, self.cur_lang_code]
            else:
                 snake_case_					       =					[self.cur_lang_code]
                 snake_case_					       =					[self.eos_token_id]

            snake_case_					       =					self.convert_ids_to_tokens(self.prefix_tokens)
            snake_case_					       =					self.convert_ids_to_tokens(self.suffix_tokens)

            snake_case_					       =					processors.TemplateProcessing(
                single=prefix_tokens_str + ['$A'] + suffix_tokens_str,							pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str,							special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str,							self.prefix_tokens + self.suffix_tokens)),							)







       def 		a_						( self,							lowerCAmelCase__)      ->      None:
            snake_case_					       =					self.convert_tokens_to_ids(lowerCAmelCase__)
            if self.legacy_behaviour:
                 snake_case_					       =					[]
                 snake_case_					       =					[self.eos_token_id, self.cur_lang_code]
            else:
                 snake_case_					       =					[self.cur_lang_code]
                 snake_case_					       =					[self.eos_token_id]

            snake_case_					       =					self.convert_ids_to_tokens(self.prefix_tokens)
            snake_case_					       =					self.convert_ids_to_tokens(self.suffix_tokens)

            snake_case_					       =					processors.TemplateProcessing(
                single=prefix_tokens_str + ['$A'] + suffix_tokens_str,							pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str,							special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str,							self.prefix_tokens + self.suffix_tokens)),							)







       def 		a_						( self,							lowerCAmelCase__,							lowerCAmelCase__ = None)      ->      Tuple[str]:
            if not self.can_save_slow_tokenizer:
                 raise ValueError(
                     'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
                     'tokenizer.')

            if not os.path.isdir(lowerCAmelCase__):
                 logger.error(f'Vocabulary path ({save_directory}) should be a directory.')
                 return
            snake_case_					       =					os.path.join(
                lowerCAmelCase__,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])

            if os.path.abspath(self.vocab_file) != os.path.abspath(lowerCAmelCase__):
                 copyfile(self.vocab_file,							lowerCAmelCase__)

            return (out_vocab_file,)



'''simple docstring'''




from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast
from ...utils import logging


lowercase			:				List[str]    		=			logging.get_logger(__name__)

lowercase			:				str    		=			{
    'EleutherAI/gpt-neo-1.3B': 'https://huggingface.co/EleutherAI/gpt-neo-1.3B/resolve/main/config.json',
    # See all GPTNeo models at https://huggingface.co/models?filter=gpt_neo
}





class  A				(							__snake_case						):
     __magic_name__						=		'''gpt_neo'''
     __magic_name__						=		['''past_key_values''']
     __magic_name__						=		{'''num_attention_heads''': '''num_heads''', '''num_hidden_layers''': '''num_layers'''}





     def __init__(							self						,			SCREAMING_SNAKE_CASE=50257						,			SCREAMING_SNAKE_CASE=2048						,			SCREAMING_SNAKE_CASE=2048						,			SCREAMING_SNAKE_CASE=24						,			SCREAMING_SNAKE_CASE=[[["global", "local"], 12]]						,			SCREAMING_SNAKE_CASE=16						,			SCREAMING_SNAKE_CASE=None						,			SCREAMING_SNAKE_CASE=256						,			SCREAMING_SNAKE_CASE="gelu_new"						,			SCREAMING_SNAKE_CASE=0.0						,			SCREAMING_SNAKE_CASE=0.0						,			SCREAMING_SNAKE_CASE=0.0						,			SCREAMING_SNAKE_CASE=0.1						,			SCREAMING_SNAKE_CASE=1e-5						,			SCREAMING_SNAKE_CASE=0.02						,			SCREAMING_SNAKE_CASE=True						,			SCREAMING_SNAKE_CASE=50256						,			SCREAMING_SNAKE_CASE=50256						,			**SCREAMING_SNAKE_CASE						,			) ->				int:






         """simple docstring"""






         A       :   Union[str, Any]					=						vocab_size
         A       :   Optional[Any]					=						max_position_embeddings
         A       :   Dict					=						hidden_size
         A       :   Optional[Any]					=						num_layers
         A       :   Tuple					=						num_heads
         A       :   int					=						intermediate_size
         A       :   Optional[Any]					=						window_size
         A       :   List[Any]					=						activation_function
         A       :   Union[str, Any]					=						resid_dropout
         A       :   Any					=						embed_dropout
         A       :   List[Any]					=						attention_dropout
         A       :   str					=						classifier_dropout
         A       :   List[Any]					=						layer_norm_epsilon
         A       :   str					=						initializer_range
         A       :   List[str]					=						use_cache

         A       :   Optional[int]					=						bos_token_id
         A       :   List[Any]					=						eos_token_id

         A       :   int					=						attention_types
         A       :   int					=						self.expand_attention_types_params(SCREAMING_SNAKE_CASE	)

         if len(self.attention_layers	) != self.num_layers:
             raise ValueError(
                 '''Configuration for convolutional module is incorrect. '''
                 '''It is required that `len(config.attention_layers)` == `config.num_layers` '''
                 F'but is `len(config.attention_layers) = {len(self.attention_layers	)}`, '
                 F'`config.num_layers = {self.num_layers}`. '
                 '''`config.attention_layers` is prepared using `config.attention_types`. '''
                 '''Please verify the value of `config.attention_types` argument.'''	)

         super().__init__(bos_token_id=SCREAMING_SNAKE_CASE						,			eos_token_id=SCREAMING_SNAKE_CASE						,			**SCREAMING_SNAKE_CASE	)





     @staticmethod
     def 				__lowerCAmelCase							(							SCREAMING_SNAKE_CASE	) ->				int:






         """simple docstring"""






         A       :   List[str]					=						[]
         for item in attention_types:
             for _ in range(item[1]	):
                 attentions.extend(item[0]	)
         return attentions







def 		lowerCAmelCase_				(   snake_case__		,				snake_case__		,				snake_case__		,				snake_case__						):




    '''simple docstring'''



    import torch

    A       :   Tuple					=						input.size()
    A       :   Union[str, Any]					=						len(snake_case__						)
    A       :   List[str]					=						shape[dimension]

    A       :   Union[str, Any]					=						torch.arange(0		,				snake_case__		,				snake_case__						)
    A       :   List[str]					=						torch.div(sizedim - size		,				snake_case__		,				rounding_mode='''floor'''						) + 1
    A       :   Optional[int]					=						torch.arange(snake_case__						) + low_indices[:min_length][:, None]

    A       :   str					=						[slice(snake_case__						)] * rank
    A       :   List[Any]					=						indices
    A       :   Union[str, Any]					=						input[s]

    A       :   List[str]					=						list(range(0		,				rank + 1						)						)
    perm.append(perm.pop(dimension + 1						)						)

    return sliced.permute(snake_case__						)







def 		lowerCAmelCase_				(   snake_case__		,				snake_case__						):




    '''simple docstring'''



    import torch

    A       :   List[str]					=						torch.arange(1		,				snake_case__						)
    A       :   Optional[int]					=						torch.remainder(snake_case__		,				snake_case__						)
    A       :   Optional[int]					=						remainders == 0
    A       :   Optional[Any]					=						candidates[divisor_indices]
    A       :   Optional[int]					=						torch.max(snake_case__						)
    return largest_divisor, torch.div(snake_case__		,				snake_case__		,				rounding_mode='''floor'''						)







class  A				(							__snake_case						):





     @property
     def 				__lowerCAmelCase							(							self	) ->				Mapping[str, Mapping[int, str]]:






         """simple docstring"""






         A       :   Tuple					=						OrderedDict({'''input_ids''': {0: '''batch''', 1: '''sequence'''}}	)
         if self.use_past:
             self.fill_with_past_key_values_(SCREAMING_SNAKE_CASE						,			direction='''inputs'''	)
             A       :   Optional[Any]					=						{0: '''batch''', 1: '''past_sequence + sequence'''}
         else:
             A       :   Dict					=						{0: '''batch''', 1: '''sequence'''}

         return common_inputs





     @property
     def 				__lowerCAmelCase							(							self	) ->				int:






         """simple docstring"""






         return self._config.num_heads





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE = -1						,			SCREAMING_SNAKE_CASE = -1						,			SCREAMING_SNAKE_CASE = False						,			SCREAMING_SNAKE_CASE = None						,			) ->				Mapping[str, Any]:






         """simple docstring"""






         A       :   List[str]					=						super(SCREAMING_SNAKE_CASE						,			self	).generate_dummy_inputs(
             SCREAMING_SNAKE_CASE						,			batch_size=SCREAMING_SNAKE_CASE						,			seq_length=SCREAMING_SNAKE_CASE						,			is_pair=SCREAMING_SNAKE_CASE						,			framework=SCREAMING_SNAKE_CASE	)

         # We need to order the input in the way they appears in the forward()
         A       :   Any					=						OrderedDict({'''input_ids''': common_inputs['''input_ids''']}	)

         # Need to add the past_keys
         if self.use_past:
             if not is_torch_available():
                 raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''	)
             else:
                 import torch

                 A,  A       :   Dict					=						common_inputs['''input_ids'''].shape
                 # Not using the same length for past_key_values
                 A       :   str					=						seqlen + 2
                 A       :   List[Any]					=						(
                     batch,
                     self.num_attention_heads,
                     past_key_values_length,
                     self._config.hidden_size // self.num_attention_heads,
                 )
                 A       :   Any					=						[
                     (torch.zeros(SCREAMING_SNAKE_CASE	), torch.zeros(SCREAMING_SNAKE_CASE	)) for _ in range(self.num_layers	)
                 ]

         A       :   str					=						common_inputs['''attention_mask''']
         if self.use_past:
             A       :   Optional[int]					=						ordered_inputs['''attention_mask'''].dtype
             A       :   List[str]					=						torch.cat(
                 [ordered_inputs['''attention_mask'''], torch.ones(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			dtype=SCREAMING_SNAKE_CASE	)]						,			dim=1	)

         return ordered_inputs





     @property
     def 				__lowerCAmelCase							(							self	) ->				int:






         """simple docstring"""






         return 13




