


import collections
import inspect
import unittest
from typing import Dict, List, Tuple

from transformers import MaskFormerSwinConfig
from transformers.testing_utils import require_torch, require_torch_multi_gpu, torch_device
from transformers.utils import is_torch_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch
    from torch import nn

    from transformers import MaskFormerSwinBackbone
    from transformers.models.maskformer import MaskFormerSwinModel







class      lowerCAmelCase__       :




     '''simple docstring'''


     def __init__(						self ,						__lowerCamelCase ,						__lowerCamelCase=1_3 ,						__lowerCamelCase=3_2 ,						__lowerCamelCase=2 ,						__lowerCamelCase=3 ,						__lowerCamelCase=1_6 ,						__lowerCamelCase=[1, 2, 1] ,						__lowerCamelCase=[2, 2, 4] ,						__lowerCamelCase=2 ,						__lowerCamelCase=2.0 ,						__lowerCamelCase=True ,						__lowerCamelCase=0.0 ,						__lowerCamelCase=0.0 ,						__lowerCamelCase=0.1 ,						__lowerCamelCase="gelu" ,						__lowerCamelCase=False ,						__lowerCamelCase=True ,						__lowerCamelCase=0.0_2 ,						__lowerCamelCase=1e-5 ,						__lowerCamelCase=True ,						__lowerCamelCase=None ,						__lowerCamelCase=True ,						__lowerCamelCase=1_0 ,						__lowerCamelCase=8 ,						__lowerCamelCase=["stage1", "stage2", "stage3"] ,						__lowerCamelCase=[1, 2, 3] ,						)			->				Optional[Any]:
            _A      :       int				    = parent
            _A      :       Optional[Any]				    = batch_size
            _A      :       str				    = image_size
            _A      :       Tuple				    = patch_size
            _A      :       Tuple				    = num_channels
            _A      :       Optional[int]				    = embed_dim
            _A      :       Dict				    = depths
            _A      :       Any				    = num_heads
            _A      :       Any				    = window_size
            _A      :       int				    = mlp_ratio
            _A      :       Any				    = qkv_bias
            _A      :       Union[str, Any]				    = hidden_dropout_prob
            _A      :       Optional[Any]				    = attention_probs_dropout_prob
            _A      :       Dict				    = drop_path_rate
            _A      :       List[Any]				    = hidden_act
            _A      :       Any				    = use_absolute_embeddings
            _A      :       Optional[int]				    = patch_norm
            _A      :       Tuple				    = layer_norm_eps
            _A      :       List[str]				    = initializer_range
            _A      :       Optional[int]				    = is_training
            _A      :       Optional[Any]				    = scope
            _A      :       Optional[int]				    = use_labels
            _A      :       Dict				    = type_sequence_label_size
            _A      :       str				    = encoder_stride
            _A      :       Optional[int]				    = out_features
            _A      :       Optional[int]				    = out_indices


     def  _lowerCamelCase (						self)			->				Union[str, Any]:
            _A      :       List[str]				    = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

            _A      :       Optional[Any]				    = None
            if self.use_labels:
                   _A      :       Any				    = ids_tensor([self.batch_size] ,						self.type_sequence_label_size)

            _A      :       Optional[int]				    = self.get_config()

            return config, pixel_values, labels


     def  _lowerCamelCase (						self)			->				Union[str, Any]:
            return MaskFormerSwinConfig(
                image_size=self.image_size ,						patch_size=self.patch_size ,						num_channels=self.num_channels ,						embed_dim=self.embed_dim ,						depths=self.depths ,						num_heads=self.num_heads ,						window_size=self.window_size ,						mlp_ratio=self.mlp_ratio ,						qkv_bias=self.qkv_bias ,						hidden_dropout_prob=self.hidden_dropout_prob ,						attention_probs_dropout_prob=self.attention_probs_dropout_prob ,						drop_path_rate=self.drop_path_rate ,						hidden_act=self.hidden_act ,						use_absolute_embeddings=self.use_absolute_embeddings ,						path_norm=self.patch_norm ,						layer_norm_eps=self.layer_norm_eps ,						initializer_range=self.initializer_range ,						encoder_stride=self.encoder_stride ,						out_features=self.out_features ,						out_indices=self.out_indices ,						)


     def  _lowerCamelCase (						self ,						__lowerCamelCase ,						__lowerCamelCase ,						__lowerCamelCase)			->				List[Any]:
            _A      :       Dict				    = MaskFormerSwinModel(config=__lowerCamelCase)
            model.to(__lowerCamelCase)
            model.eval()
            _A      :       int				    = model(__lowerCamelCase)

            _A      :       Any				    = ((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths) - 1))
            _A      :       List[str]				    = int(config.embed_dim * 2 ** (len(config.depths) - 1))

            self.parent.assertEqual(result.last_hidden_state.shape ,						(self.batch_size, expected_seq_len, expected_dim))


     def  _lowerCamelCase (						self ,						__lowerCamelCase ,						__lowerCamelCase ,						__lowerCamelCase)			->				Dict:
            _A      :       Optional[Any]				    = MaskFormerSwinBackbone(config=__lowerCamelCase)
            model.to(__lowerCamelCase)
            model.eval()
            _A      :       Dict				    = model(__lowerCamelCase)

            # verify feature maps
            self.parent.assertEqual(len(result.feature_maps) ,						len(config.out_features))
            self.parent.assertListEqual(list(result.feature_maps[0].shape) ,						[1_3, 1_6, 1_6, 1_6])

            # verify channels
            self.parent.assertEqual(len(model.channels) ,						len(config.out_features))
            self.parent.assertListEqual(model.channels ,						[1_6, 3_2, 6_4])

            # verify ValueError
            with self.parent.assertRaises(__lowerCamelCase):
                   _A      :       Union[str, Any]				    = ["stem"]
                   _A      :       Union[str, Any]				    = MaskFormerSwinBackbone(config=__lowerCamelCase)


     def  _lowerCamelCase (						self)			->				Dict:
            _A      :       Any				    = self.prepare_config_and_inputs()
            _A    ,			_A    ,			_A      :       List[Any]				    = config_and_inputs
            _A      :       Optional[int]				    = {"pixel_values": pixel_values}
            return config, inputs_dict







@require_torch
class      lowerCAmelCase__       (				a     ,    a     ,    unittest.TestCase):




     '''simple docstring'''
     __SCREAMING_SNAKE_CASE  			= (
         (
             MaskFormerSwinModel,
             MaskFormerSwinBackbone,
         )
         if is_torch_available()
         else ()
     )
     __SCREAMING_SNAKE_CASE  			= {"feature-extraction": MaskFormerSwinModel} if is_torch_available() else {}
     __SCREAMING_SNAKE_CASE  			= False
     __SCREAMING_SNAKE_CASE  			= False
     __SCREAMING_SNAKE_CASE  			= False
     __SCREAMING_SNAKE_CASE  			= False
     __SCREAMING_SNAKE_CASE  			= False


     def  _lowerCamelCase (						self)			->				str:
            _A      :       Union[str, Any]				    = MaskFormerSwinModelTester(self)
            _A      :       Optional[int]				    = ConfigTester(self ,						config_class=__lowerCamelCase ,						embed_dim=3_7)


     @require_torch_multi_gpu
     @unittest.skip(
         reason=(
             "`MaskFormerSwinModel` outputs `hidden_states_spatial_dimensions` which doesn't work well with"
             " `nn.DataParallel`"
         ))
     def  _lowerCamelCase (						self)			->				Union[str, Any]:
            pass


     def  _lowerCamelCase (						self)			->				int:
            self.create_and_test_config_common_properties()
            self.config_tester.create_and_test_config_to_json_string()
            self.config_tester.create_and_test_config_to_json_file()
            self.config_tester.create_and_test_config_from_and_save_pretrained()
            self.config_tester.create_and_test_config_with_num_labels()
            self.config_tester.check_config_can_be_init_without_params()
            self.config_tester.check_config_arguments_init()


     def  _lowerCamelCase (						self)			->				str:
            return


     def  _lowerCamelCase (						self)			->				List[Any]:
            _A      :       Optional[int]				    = self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_model(*__lowerCamelCase)


     def  _lowerCamelCase (						self)			->				Union[str, Any]:
            _A      :       Any				    = self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_backbone(*__lowerCamelCase)


     @unittest.skip("Swin does not use inputs_embeds")
     def  _lowerCamelCase (						self)			->				str:
            pass


     @unittest.skip("Swin does not support feedforward chunking")
     def  _lowerCamelCase (						self)			->				List[Any]:
            pass


     def  _lowerCamelCase (						self)			->				Optional[int]:
            _A    ,			_A      :       Union[str, Any]				    = self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                   _A      :       Union[str, Any]				    = model_class(__lowerCamelCase)
                   self.assertIsInstance(model.get_input_embeddings() ,						(nn.Module))
                   _A      :       Dict				    = model.get_output_embeddings()
                   self.assertTrue(x is None or isinstance(__lowerCamelCase ,						nn.Linear))


     def  _lowerCamelCase (						self)			->				Any:
            _A    ,			_A      :       List[Any]				    = self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                   _A      :       int				    = model_class(__lowerCamelCase)
                   _A      :       Optional[int]				    = inspect.signature(model.forward)
                   # signature.parameters is an OrderedDict => so arg_names order is deterministic
                   _A      :       int				    = [*signature.parameters.keys()]

                   _A      :       Optional[int]				    = ["pixel_values"]
                   self.assertListEqual(arg_names[:1] ,						__lowerCamelCase)


     @unittest.skip(reason="MaskFormerSwin is only used as backbone and doesn't support output_attentions")
     def  _lowerCamelCase (						self)			->				Tuple:
            pass


     @unittest.skip(reason="MaskFormerSwin is only used as an internal backbone")
     def  _lowerCamelCase (						self)			->				str:
            pass


     def  _lowerCamelCase (						self ,						__lowerCamelCase ,						__lowerCamelCase ,						__lowerCamelCase ,						__lowerCamelCase)			->				Optional[int]:
            _A      :       Any				    = model_class(__lowerCamelCase)
            model.to(__lowerCamelCase)
            model.eval()

            with torch.no_grad():
                   _A      :       str				    = model(**self._prepare_for_class(__lowerCamelCase ,						__lowerCamelCase))

            _A      :       Tuple				    = outputs.hidden_states

            _A      :       Any				    = getattr(
                self.model_tester ,						"expected_num_hidden_layers" ,						len(self.model_tester.depths) + 1)
            self.assertEqual(len(__lowerCamelCase) ,						__lowerCamelCase)

            # Swin has a different seq_length
            _A      :       Optional[int]				    = (
                config.patch_size
                if isinstance(config.patch_size ,						collections.abc.Iterable)
                else (config.patch_size, config.patch_size)
            )

            _A      :       str				    = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

            self.assertListEqual(
                list(hidden_states[0].shape[-2:]) ,						[num_patches, self.model_tester.embed_dim] ,						)


     def  _lowerCamelCase (						self)			->				Dict:
            _A    ,			_A      :       List[str]				    = self.model_tester.prepare_config_and_inputs_for_common()

            _A      :       Any				    = (
                self.model_tester.image_size
                if isinstance(self.model_tester.image_size ,						collections.abc.Iterable)
                else (self.model_tester.image_size, self.model_tester.image_size)
            )

            for model_class in self.all_model_classes:
                   _A      :       List[Any]				    = True
                   self.check_hidden_states_output(__lowerCamelCase ,						__lowerCamelCase ,						__lowerCamelCase ,						__lowerCamelCase)

                   # check that output_hidden_states also work using config
                   del inputs_dict["output_hidden_states"]
                   _A      :       Optional[int]				    = True

                   self.check_hidden_states_output(__lowerCamelCase ,						__lowerCamelCase ,						__lowerCamelCase ,						__lowerCamelCase)


     def  _lowerCamelCase (						self)			->				Tuple:
            _A    ,			_A      :       List[str]				    = self.model_tester.prepare_config_and_inputs_for_common()
            _A      :       Optional[int]				    = 3

            _A      :       Dict				    = (
                self.model_tester.image_size
                if isinstance(self.model_tester.image_size ,						collections.abc.Iterable)
                else (self.model_tester.image_size, self.model_tester.image_size)
            )
            _A      :       Optional[int]				    = (
                config.patch_size
                if isinstance(config.patch_size ,						collections.abc.Iterable)
                else (config.patch_size, config.patch_size)
            )

            _A      :       int				    = image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
            _A      :       Dict				    = image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

            for model_class in self.all_model_classes:
                   _A      :       List[Any]				    = True
                   self.check_hidden_states_output(__lowerCamelCase ,						__lowerCamelCase ,						__lowerCamelCase ,						(padded_height, padded_width))

                   # check that output_hidden_states also work using config
                   del inputs_dict["output_hidden_states"]
                   _A      :       Union[str, Any]				    = True
                   self.check_hidden_states_output(__lowerCamelCase ,						__lowerCamelCase ,						__lowerCamelCase ,						(padded_height, padded_width))


     @unittest.skip(reason="MaskFormerSwin doesn't have pretrained checkpoints")
     def  _lowerCamelCase (						self)			->				List[str]:
            pass


     @unittest.skip(reason="This will be fixed once MaskFormerSwin is replaced by native Swin")
     def  _lowerCamelCase (						self)			->				List[str]:
            pass


     @unittest.skip(reason="This will be fixed once MaskFormerSwin is replaced by native Swin")
     def  _lowerCamelCase (						self)			->				str:
            pass


     def  _lowerCamelCase (						self)			->				Optional[Any]:
            _A    ,			_A      :       Any				    = self.model_tester.prepare_config_and_inputs_for_common()

            def set_nan_tensor_to_zero(__lowerCamelCase):
                   _A      :       Optional[int]				    = 0
                   return t

            def check_equivalence(__lowerCamelCase ,						__lowerCamelCase ,						__lowerCamelCase ,						__lowerCamelCase={}):
                   with torch.no_grad():
                          _A      :       Any				    = model(**__lowerCamelCase ,						return_dict=__lowerCamelCase ,						**__lowerCamelCase)
                          _A      :       int				    = model(**__lowerCamelCase ,						return_dict=__lowerCamelCase ,						**__lowerCamelCase).to_tuple()

                          def recursive_check(__lowerCamelCase ,						__lowerCamelCase):
                                 if isinstance(__lowerCamelCase ,						(List, Tuple)):
                                        for tuple_iterable_value, dict_iterable_value in zip(__lowerCamelCase ,						__lowerCamelCase):
                                               recursive_check(__lowerCamelCase ,						__lowerCamelCase)
                                 elif isinstance(__lowerCamelCase ,						__lowerCamelCase):
                                        for tuple_iterable_value, dict_iterable_value in zip(
                                            tuple_object.values() ,						dict_object.values()):
                                               recursive_check(__lowerCamelCase ,						__lowerCamelCase)
                                 elif tuple_object is None:
                                        return
                                 else:
                                        self.assertTrue(
                                            torch.allclose(
                                                set_nan_tensor_to_zero(__lowerCamelCase) ,						set_nan_tensor_to_zero(__lowerCamelCase) ,						atol=1e-5) ,						msg=(
                                                "Tuple and dict output are not equal. Difference:"
                                                F" {torch.max(torch.abs(tuple_object - dict_object))}. Tuple has `nan`:"
                                                F" {torch.isnan(__lowerCamelCase).any()} and `inf`: {torch.isinf(__lowerCamelCase)}. Dict has"
                                                F" `nan`: {torch.isnan(__lowerCamelCase).any()} and `inf`: {torch.isinf(__lowerCamelCase)}."
                                            ) ,						)

                          recursive_check(__lowerCamelCase ,						__lowerCamelCase)

            for model_class in self.all_model_classes:
                   _A      :       List[Any]				    = model_class(__lowerCamelCase)
                   model.to(__lowerCamelCase)
                   model.eval()

                   _A      :       str				    = self._prepare_for_class(__lowerCamelCase ,						__lowerCamelCase)
                   _A      :       Tuple				    = self._prepare_for_class(__lowerCamelCase ,						__lowerCamelCase)
                   check_equivalence(__lowerCamelCase ,						__lowerCamelCase ,						__lowerCamelCase)

                   _A      :       Any				    = self._prepare_for_class(__lowerCamelCase ,						__lowerCamelCase ,						return_labels=__lowerCamelCase)
                   _A      :       List[Any]				    = self._prepare_for_class(__lowerCamelCase ,						__lowerCamelCase ,						return_labels=__lowerCamelCase)
                   check_equivalence(__lowerCamelCase ,						__lowerCamelCase ,						__lowerCamelCase)

                   _A      :       List[Any]				    = self._prepare_for_class(__lowerCamelCase ,						__lowerCamelCase)
                   _A      :       str				    = self._prepare_for_class(__lowerCamelCase ,						__lowerCamelCase)
                   check_equivalence(__lowerCamelCase ,						__lowerCamelCase ,						__lowerCamelCase ,						{"output_hidden_states": True})

                   _A      :       Union[str, Any]				    = self._prepare_for_class(__lowerCamelCase ,						__lowerCamelCase ,						return_labels=__lowerCamelCase)
                   _A      :       Optional[Any]				    = self._prepare_for_class(__lowerCamelCase ,						__lowerCamelCase ,						return_labels=__lowerCamelCase)
                   check_equivalence(__lowerCamelCase ,						__lowerCamelCase ,						__lowerCamelCase ,						{"output_hidden_states": True})







@require_torch
class      lowerCAmelCase__       (				unittest.TestCase     ,    a):




     '''simple docstring'''
     __SCREAMING_SNAKE_CASE  			= (MaskFormerSwinBackbone,) if is_torch_available() else ()
     __SCREAMING_SNAKE_CASE  			= MaskFormerSwinConfig


     def  _lowerCamelCase (						self)			->				Optional[Any]:
            _A      :       Tuple				    = MaskFormerSwinModelTester(self)


     def  _lowerCamelCase (						self)			->				Optional[Any]:
            _A    ,			_A      :       List[Any]				    = self.model_tester.prepare_config_and_inputs_for_common()
            _A      :       Union[str, Any]				    = inputs_dict["pixel_values"].shape[0]

            for backbone_class in self.all_model_classes:
                   _A      :       Optional[Any]				    = backbone_class(__lowerCamelCase)
                   backbone.to(__lowerCamelCase)
                   backbone.eval()

                   _A      :       List[Any]				    = backbone(**__lowerCamelCase)

                   # Test default outputs and verify feature maps
                   self.assertIsInstance(outputs.feature_maps ,						__lowerCamelCase)
                   self.assertTrue(len(outputs.feature_maps) == len(backbone.channels))
                   for feature_map, n_channels in zip(outputs.feature_maps ,						backbone.channels):
                          self.assertTrue(feature_map.shape[:2] ,						(batch_size, n_channels))
                   self.assertIsNone(outputs.hidden_states)
                   self.assertIsNone(outputs.attentions)

                   # Test output_hidden_states=True
                   _A      :       List[str]				    = backbone(**__lowerCamelCase ,						output_hidden_states=__lowerCamelCase)
                   self.assertIsNotNone(outputs.hidden_states)
                   self.assertTrue(len(outputs.hidden_states) ,						len(backbone.stage_names))
                   # We skip the stem layer
                   for hidden_states, n_channels in zip(outputs.hidden_states[1:] ,						backbone.channels):
                          for hidden_state in hidden_states:
                                 # Hidden states are in the format (batch_size, (height * width), n_channels)
                                 _A    ,			_A    ,			_A      :       List[str]				    = hidden_state.shape
                                 self.assertTrue((h_batch_size, h_n_channels) ,						(batch_size, n_channels))

            # Test output_attentions=True
                   if self.has_attentions:
                          _A      :       int				    = backbone(**__lowerCamelCase ,						output_attentions=__lowerCamelCase)
                          self.assertIsNotNone(outputs.attentions)




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
    from .tokenization_nllb import NllbTokenizer
else:
    lowerCAmelCase__      		= None


lowerCAmelCase__      		= logging.get_logger(__name__)


lowerCAmelCase__      		= {'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

lowerCAmelCase__      		= {
    'vocab_file': {
        'facebook/nllb-200-distilled-600M': (
            'https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/nllb-200-distilled-600M': (
            'https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json'
        ),
    },
}

lowerCAmelCase__      		= {
    'facebook/nllb-large-en-ro': 10_24,
    'facebook/nllb-200-distilled-600M': 10_24,
}

# fmt: off
lowerCAmelCase__      		= ['ace_Arab', 'ace_Latn', 'acm_Arab', 'acq_Arab', 'aeb_Arab', 'afr_Latn', 'ajp_Arab', 'aka_Latn', 'amh_Ethi', 'apc_Arab', 'arb_Arab', 'ars_Arab', 'ary_Arab', 'arz_Arab', 'asm_Beng', 'ast_Latn', 'awa_Deva', 'ayr_Latn', 'azb_Arab', 'azj_Latn', 'bak_Cyrl', 'bam_Latn', 'ban_Latn', 'bel_Cyrl', 'bem_Latn', 'ben_Beng', 'bho_Deva', 'bjn_Arab', 'bjn_Latn', 'bod_Tibt', 'bos_Latn', 'bug_Latn', 'bul_Cyrl', 'cat_Latn', 'ceb_Latn', 'ces_Latn', 'cjk_Latn', 'ckb_Arab', 'crh_Latn', 'cym_Latn', 'dan_Latn', 'deu_Latn', 'dik_Latn', 'dyu_Latn', 'dzo_Tibt', 'ell_Grek', 'eng_Latn', 'epo_Latn', 'est_Latn', 'eus_Latn', 'ewe_Latn', 'fao_Latn', 'pes_Arab', 'fij_Latn', 'fin_Latn', 'fon_Latn', 'fra_Latn', 'fur_Latn', 'fuv_Latn', 'gla_Latn', 'gle_Latn', 'glg_Latn', 'grn_Latn', 'guj_Gujr', 'hat_Latn', 'hau_Latn', 'heb_Hebr', 'hin_Deva', 'hne_Deva', 'hrv_Latn', 'hun_Latn', 'hye_Armn', 'ibo_Latn', 'ilo_Latn', 'ind_Latn', 'isl_Latn', 'ita_Latn', 'jav_Latn', 'jpn_Jpan', 'kab_Latn', 'kac_Latn', 'kam_Latn', 'kan_Knda', 'kas_Arab', 'kas_Deva', 'kat_Geor', 'knc_Arab', 'knc_Latn', 'kaz_Cyrl', 'kbp_Latn', 'kea_Latn', 'khm_Khmr', 'kik_Latn', 'kin_Latn', 'kir_Cyrl', 'kmb_Latn', 'kon_Latn', 'kor_Hang', 'kmr_Latn', 'lao_Laoo', 'lvs_Latn', 'lij_Latn', 'lim_Latn', 'lin_Latn', 'lit_Latn', 'lmo_Latn', 'ltg_Latn', 'ltz_Latn', 'lua_Latn', 'lug_Latn', 'luo_Latn', 'lus_Latn', 'mag_Deva', 'mai_Deva', 'mal_Mlym', 'mar_Deva', 'min_Latn', 'mkd_Cyrl', 'plt_Latn', 'mlt_Latn', 'mni_Beng', 'khk_Cyrl', 'mos_Latn', 'mri_Latn', 'zsm_Latn', 'mya_Mymr', 'nld_Latn', 'nno_Latn', 'nob_Latn', 'npi_Deva', 'nso_Latn', 'nus_Latn', 'nya_Latn', 'oci_Latn', 'gaz_Latn', 'ory_Orya', 'pag_Latn', 'pan_Guru', 'pap_Latn', 'pol_Latn', 'por_Latn', 'prs_Arab', 'pbt_Arab', 'quy_Latn', 'ron_Latn', 'run_Latn', 'rus_Cyrl', 'sag_Latn', 'san_Deva', 'sat_Beng', 'scn_Latn', 'shn_Mymr', 'sin_Sinh', 'slk_Latn', 'slv_Latn', 'smo_Latn', 'sna_Latn', 'snd_Arab', 'som_Latn', 'sot_Latn', 'spa_Latn', 'als_Latn', 'srd_Latn', 'srp_Cyrl', 'ssw_Latn', 'sun_Latn', 'swe_Latn', 'swh_Latn', 'szl_Latn', 'tam_Taml', 'tat_Cyrl', 'tel_Telu', 'tgk_Cyrl', 'tgl_Latn', 'tha_Thai', 'tir_Ethi', 'taq_Latn', 'taq_Tfng', 'tpi_Latn', 'tsn_Latn', 'tso_Latn', 'tuk_Latn', 'tum_Latn', 'tur_Latn', 'twi_Latn', 'tzm_Tfng', 'uig_Arab', 'ukr_Cyrl', 'umb_Latn', 'urd_Arab', 'uzn_Latn', 'vec_Latn', 'vie_Latn', 'war_Latn', 'wol_Latn', 'xho_Latn', 'ydd_Hebr', 'yor_Latn', 'yue_Hant', 'zho_Hans', 'zho_Hant', 'zul_Latn']







class      lowerCAmelCase__       (				a):




     '''simple docstring'''
     __SCREAMING_SNAKE_CASE  			= VOCAB_FILES_NAMES
     __SCREAMING_SNAKE_CASE  			= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __SCREAMING_SNAKE_CASE  			= PRETRAINED_VOCAB_FILES_MAP
     __SCREAMING_SNAKE_CASE  			= ["input_ids", "attention_mask"]
     __SCREAMING_SNAKE_CASE  			= NllbTokenizer

     __SCREAMING_SNAKE_CASE  			= []
     __SCREAMING_SNAKE_CASE  			= []


     def __init__(						self ,						__lowerCamelCase=None ,						__lowerCamelCase=None ,						__lowerCamelCase="<s>" ,						__lowerCamelCase="</s>" ,						__lowerCamelCase="</s>" ,						__lowerCamelCase="<s>" ,						__lowerCamelCase="<unk>" ,						__lowerCamelCase="<pad>" ,						__lowerCamelCase="<mask>" ,						__lowerCamelCase=None ,						__lowerCamelCase=None ,						__lowerCamelCase=None ,						__lowerCamelCase=False ,						**__lowerCamelCase ,						)			->				Tuple:
            # Mask token behave like a normal word, i.e. include the space before it
            _A      :       Any				    = AddedToken(__lowerCamelCase ,						lstrip=__lowerCamelCase ,						rstrip=__lowerCamelCase) if isinstance(__lowerCamelCase ,						__lowerCamelCase) else mask_token
            _A      :       Optional[int]				    = legacy_behaviour
            super().__init__(
                vocab_file=__lowerCamelCase ,						tokenizer_file=__lowerCamelCase ,						bos_token=__lowerCamelCase ,						eos_token=__lowerCamelCase ,						sep_token=__lowerCamelCase ,						cls_token=__lowerCamelCase ,						unk_token=__lowerCamelCase ,						pad_token=__lowerCamelCase ,						mask_token=__lowerCamelCase ,						src_lang=__lowerCamelCase ,						tgt_lang=__lowerCamelCase ,						additional_special_tokens=__lowerCamelCase ,						legacy_behaviour=__lowerCamelCase ,						**__lowerCamelCase ,						)

            _A      :       int				    = vocab_file
            _A      :       Optional[Any]				    = False if not self.vocab_file else True

            _A      :       Tuple				    = FAIRSEQ_LANGUAGE_CODES.copy()

            if additional_special_tokens is not None:
                   # Only add those special tokens if they are not already there.
                   _additional_special_tokens.extend(
                       [t for t in additional_special_tokens if t not in _additional_special_tokens])

            self.add_special_tokens({"additional_special_tokens": _additional_special_tokens})
            _A      :       Union[str, Any]				    = {
                lang_code: self.convert_tokens_to_ids(__lowerCamelCase) for lang_code in FAIRSEQ_LANGUAGE_CODES
            }

            _A      :       Optional[int]				    = src_lang if src_lang is not None else "eng_Latn"
            _A      :       Union[str, Any]				    = self.convert_tokens_to_ids(self._src_lang)
            _A      :       List[str]				    = tgt_lang
            self.set_src_lang_special_tokens(self._src_lang)


     @property
     def  _lowerCamelCase (						self)			->				str:
            return self._src_lang


     @src_lang.setter
     def  _lowerCamelCase (						self ,						__lowerCamelCase)			->				None:
            _A      :       Tuple				    = new_src_lang
            self.set_src_lang_special_tokens(self._src_lang)


     def  _lowerCamelCase (						self ,						__lowerCamelCase ,						__lowerCamelCase = None)			->				List[int]:
            if token_ids_a is None:
                   return self.prefix_tokens + token_ids_a + self.suffix_tokens
            # We don't expect to process pairs, but leave the pair logic for API consistency
            return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens


     def  _lowerCamelCase (						self ,						__lowerCamelCase ,						__lowerCamelCase = None)			->				List[int]:
            _A      :       Tuple				    = [self.sep_token_id]
            _A      :       List[Any]				    = [self.cls_token_id]

            if token_ids_a is None:
                   return len(cls + token_ids_a + sep) * [0]
            return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]


     def  _lowerCamelCase (						self ,						__lowerCamelCase ,						__lowerCamelCase ,						__lowerCamelCase ,						__lowerCamelCase ,						**__lowerCamelCase)			->				Optional[int]:
            if src_lang is None or tgt_lang is None:
                   raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model")
            _A      :       List[Any]				    = src_lang
            _A      :       Optional[int]				    = self(__lowerCamelCase ,						add_special_tokens=__lowerCamelCase ,						return_tensors=__lowerCamelCase ,						**__lowerCamelCase)
            _A      :       Tuple				    = self.convert_tokens_to_ids(__lowerCamelCase)
            _A      :       Tuple				    = tgt_lang_id
            return inputs


     def  _lowerCamelCase (						self ,						__lowerCamelCase ,						__lowerCamelCase = "eng_Latn" ,						__lowerCamelCase = None ,						__lowerCamelCase = "fra_Latn" ,						**__lowerCamelCase ,						)			->				BatchEncoding:
            _A      :       Tuple				    = src_lang
            _A      :       int				    = tgt_lang
            return super().prepare_seqaseq_batch(__lowerCamelCase ,						__lowerCamelCase ,						**__lowerCamelCase)


     def  _lowerCamelCase (						self)			->				str:
            return self.set_src_lang_special_tokens(self.src_lang)


     def  _lowerCamelCase (						self)			->				List[str]:
            return self.set_tgt_lang_special_tokens(self.tgt_lang)


     def  _lowerCamelCase (						self ,						__lowerCamelCase)			->				None:
            _A      :       Dict				    = self.convert_tokens_to_ids(__lowerCamelCase)

            if self.legacy_behaviour:
                   _A      :       List[str]				    = []
                   _A      :       Dict				    = [self.eos_token_id, self.cur_lang_code]
            else:
                   _A      :       Tuple				    = [self.cur_lang_code]
                   _A      :       Optional[Any]				    = [self.eos_token_id]

            _A      :       Optional[int]				    = self.convert_ids_to_tokens(self.prefix_tokens)
            _A      :       int				    = self.convert_ids_to_tokens(self.suffix_tokens)

            _A      :       List[Any]				    = processors.TemplateProcessing(
                single=prefix_tokens_str + ["$A"] + suffix_tokens_str ,						pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str ,						special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str ,						self.prefix_tokens + self.suffix_tokens)) ,						)


     def  _lowerCamelCase (						self ,						__lowerCamelCase)			->				None:
            _A      :       Optional[Any]				    = self.convert_tokens_to_ids(__lowerCamelCase)
            if self.legacy_behaviour:
                   _A      :       Tuple				    = []
                   _A      :       Any				    = [self.eos_token_id, self.cur_lang_code]
            else:
                   _A      :       Union[str, Any]				    = [self.cur_lang_code]
                   _A      :       str				    = [self.eos_token_id]

            _A      :       Optional[Any]				    = self.convert_ids_to_tokens(self.prefix_tokens)
            _A      :       Dict				    = self.convert_ids_to_tokens(self.suffix_tokens)

            _A      :       Union[str, Any]				    = processors.TemplateProcessing(
                single=prefix_tokens_str + ["$A"] + suffix_tokens_str ,						pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str ,						special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str ,						self.prefix_tokens + self.suffix_tokens)) ,						)


     def  _lowerCamelCase (						self ,						__lowerCamelCase ,						__lowerCamelCase = None)			->				Tuple[str]:
            if not self.can_save_slow_tokenizer:
                   raise ValueError(
                       "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                       "tokenizer.")

            if not os.path.isdir(__lowerCamelCase):
                   logger.error(F"Vocabulary path ({save_directory}) should be a directory.")
                   return
            _A      :       Dict				    = os.path.join(
                __lowerCamelCase ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])

            if os.path.abspath(self.vocab_file) != os.path.abspath(__lowerCamelCase):
                   copyfile(self.vocab_file ,						__lowerCamelCase)

            return (out_vocab_file,)
