




'''simple docstring'''





import inspect
import unittest

import numpy as np

from tests.test_modeling_common import floats_tensor
from transformers import DetrConfig, MaskFormerConfig, SwinConfig, is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_torch_multi_gpu, require_vision, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import MaskFormerForInstanceSegmentation, MaskFormerModel

		if is_vision_available():
				from transformers import MaskFormerImageProcessor

if is_vision_available():
		from PIL import Image




class  _snake_case		:




			def __init__(					self				,							a__				,							a__=2				,							a__=True				,							a__=False				,							a__=10				,							a__=3				,							a__=32 * 4				,							a__=32 * 6				,							a__=4				,							a__=32				,							)				->				List[Any]:

										'''simple docstring'''




										snake_case_						=      parent
										snake_case_						=      batch_size
										snake_case_						=      is_training
										snake_case_						=      use_auxiliary_loss
										snake_case_						=      num_queries
										snake_case_						=      num_channels
										snake_case_						=      min_size
										snake_case_						=      max_size
										snake_case_						=      num_labels
										snake_case_						=      mask_feature_size




			def    lowerCAmelCase__				(					self							)				->				int:

										'''simple docstring'''




										snake_case_						=      floats_tensor([self.batch_size, self.num_channels, self.min_size, self.max_size]							).to(
										    a__							)

										snake_case_						=      torch.ones([self.batch_size, self.min_size, self.max_size]				,							device=a__							)

										snake_case_						=      (
										    torch.rand([self.batch_size, self.num_labels, self.min_size, self.max_size]				,							device=a__							) > 0.5
										).float()
										snake_case_						=      (torch.rand((self.batch_size, self.num_labels)				,							device=a__							) > 0.5).long()

										snake_case_						=      self.get_config()
										return config, pixel_values, pixel_mask, mask_labels, class_labels




			def    lowerCAmelCase__				(					self							)				->				List[str]:

										'''simple docstring'''




										return MaskFormerConfig.from_backbone_and_decoder_configs(
										    backbone_config=SwinConfig(
										        depths=[1, 1, 1, 1]				,							)				,							decoder_config=DetrConfig(
										        decoder_ffn_dim=128				,							num_queries=self.num_queries				,							decoder_attention_heads=2				,							d_model=self.mask_feature_size				,							)				,							mask_feature_size=self.mask_feature_size				,							fpn_feature_size=self.mask_feature_size				,							num_channels=self.num_channels				,							num_labels=self.num_labels				,							)




			def    lowerCAmelCase__				(					self							)				->				str:

										'''simple docstring'''




										snake_case_					,      snake_case_					,      snake_case_					,      snake_case_					,      snake_case_						=      self.prepare_config_and_inputs()
										snake_case_						=      {"pixel_values": pixel_values, "pixel_mask": pixel_mask}
										return config, inputs_dict




			def    lowerCAmelCase__				(					self				,							a__				,							a__							)				->				Optional[Any]:

										'''simple docstring'''




										snake_case_						=      output.encoder_hidden_states
										snake_case_						=      output.pixel_decoder_hidden_states
										snake_case_						=      output.transformer_decoder_hidden_states

										self.parent.assertTrue(len(a__							)				,							len(config.backbone_config.depths							)							)
										self.parent.assertTrue(len(a__							)				,							len(config.backbone_config.depths							)							)
										self.parent.assertTrue(len(a__							)				,							config.decoder_config.decoder_layers							)




			def    lowerCAmelCase__				(					self				,							a__				,							a__				,							a__				,							a__=False							)				->				int:

										'''simple docstring'''




										with torch.no_grad():
																	snake_case_						=      MaskFormerModel(config=a__							)
																	model.to(a__							)
																	model.eval()

																	snake_case_						=      model(pixel_values=a__				,							pixel_mask=a__							)
																	snake_case_						=      model(a__				,							output_hidden_states=a__							)
										# the correct shape of output.transformer_decoder_hidden_states ensure the correcteness of the
										# encoder and pixel decoder
										self.parent.assertEqual(
										    output.transformer_decoder_last_hidden_state.shape				,							(self.batch_size, self.num_queries, self.mask_feature_size)				,							)
										# let's ensure the other two hidden state exists
										self.parent.assertTrue(output.pixel_decoder_last_hidden_state is not None							)
										self.parent.assertTrue(output.encoder_last_hidden_state is not None							)

										if output_hidden_states:
																	self.check_output_hidden_state(a__				,							a__							)




			def    lowerCAmelCase__				(					self				,							a__				,							a__				,							a__				,							a__				,							a__							)				->				Union[str, Any]:

										'''simple docstring'''




										snake_case_						=      MaskFormerForInstanceSegmentation(config=a__							)
										model.to(a__							)
										model.eval()

										def comm_check_on_output(a__							):
																	# let's still check that all the required stuff is there
																	self.parent.assertTrue(result.transformer_decoder_last_hidden_state is not None							)
																	self.parent.assertTrue(result.pixel_decoder_last_hidden_state is not None							)
																	self.parent.assertTrue(result.encoder_last_hidden_state is not None							)
																	# okay, now we need to check the logits shape
																	# due to the encoder compression, masks have a //4 spatial size
																	self.parent.assertEqual(
																	    result.masks_queries_logits.shape				,							(self.batch_size, self.num_queries, self.min_size // 4, self.max_size // 4)				,							)
																	# + 1 for null class
																	self.parent.assertEqual(
																	    result.class_queries_logits.shape				,							(self.batch_size, self.num_queries, self.num_labels + 1)							)

										with torch.no_grad():
																	snake_case_						=      model(pixel_values=a__				,							pixel_mask=a__							)
																	snake_case_						=      model(a__							)

																	comm_check_on_output(a__							)

																	snake_case_						=      model(
																	    pixel_values=a__				,							pixel_mask=a__				,							mask_labels=a__				,							class_labels=a__							)

										comm_check_on_output(a__							)

										self.parent.assertTrue(result.loss is not None							)
										self.parent.assertEqual(result.loss.shape				,							torch.Size([1]							)							)




@require_torch
class  _snake_case		(	lowercase_						,						lowercase_						,						unittest.TestCase							):
			lowerCAmelCase_     :      str						       =    (MaskFormerModel, MaskFormerForInstanceSegmentation) if is_torch_available() else ()
			lowerCAmelCase_     :      Dict						       =    (
			    {"feature-extraction": MaskFormerModel, "image-segmentation": MaskFormerForInstanceSegmentation}
			    if is_torch_available()
			    else {}
			)

			lowerCAmelCase_     :      Dict						       =    False
			lowerCAmelCase_     :      List[Any]						       =    False
			lowerCAmelCase_     :      Optional[Any]						       =    False
			lowerCAmelCase_     :      List[str]						       =    False




			def    lowerCAmelCase__				(					self							)				->				str:

										'''simple docstring'''




										snake_case_						=      MaskFormerModelTester(self							)
										snake_case_						=      ConfigTester(self				,							config_class=a__				,							has_text_modality=a__							)




			def    lowerCAmelCase__				(					self							)				->				str:

										'''simple docstring'''




										self.config_tester.run_common_tests()




			def    lowerCAmelCase__				(					self							)				->				Any:

										'''simple docstring'''




										snake_case_					,      snake_case_						=      self.model_tester.prepare_config_and_inputs_for_common()
										self.model_tester.create_and_check_maskformer_model(a__				,							**a__				,							output_hidden_states=a__							)




			def    lowerCAmelCase__				(					self							)				->				Any:

										'''simple docstring'''




										snake_case_						=      self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_maskformer_instance_segmentation_head_model(*a__							)




			@unittest.skip(reason="MaskFormer does not use inputs_embeds"							)
			def    lowerCAmelCase__				(					self							)				->				Any:

										'''simple docstring'''




										pass




			@unittest.skip(reason="MaskFormer does not have a get_input_embeddings method"							)
			def    lowerCAmelCase__				(					self							)				->				Optional[int]:

										'''simple docstring'''




										pass




			@unittest.skip(reason="MaskFormer is not a generative model"							)
			def    lowerCAmelCase__				(					self							)				->				Optional[int]:

										'''simple docstring'''




										pass




			@unittest.skip(reason="MaskFormer does not use token embeddings"							)
			def    lowerCAmelCase__				(					self							)				->				str:

										'''simple docstring'''




										pass




			@require_torch_multi_gpu
			@unittest.skip(
			    reason="MaskFormer has some layers using `add_module` which doesn't work well with `nn.DataParallel`"							)
			def    lowerCAmelCase__				(					self							)				->				Optional[Any]:

										'''simple docstring'''




										pass




			@unittest.skip("Will be fixed soon by reducing the size of the model used for common tests."							)
			def    lowerCAmelCase__				(					self							)				->				List[str]:

										'''simple docstring'''




										pass




			def    lowerCAmelCase__				(					self							)				->				List[str]:

										'''simple docstring'''




										snake_case_					,      snake_case_						=      self.model_tester.prepare_config_and_inputs_for_common()

										for model_class in self.all_model_classes:
																	snake_case_						=      model_class(a__							)
																	snake_case_						=      inspect.signature(model.forward							)
																	# signature.parameters is an OrderedDict => so arg_names order is deterministic
																	snake_case_						=      [*signature.parameters.keys()]

																	snake_case_						=      ["pixel_values"]
																	self.assertListEqual(arg_names[:1]				,							a__							)




			@slow
			def    lowerCAmelCase__				(					self							)				->				Optional[Any]:

										'''simple docstring'''




										for model_name in ["facebook/maskformer-swin-small-coco"]:
																	snake_case_						=      MaskFormerModel.from_pretrained(a__							)
																	self.assertIsNotNone(a__							)




			def    lowerCAmelCase__				(					self							)				->				int:

										'''simple docstring'''




										snake_case_						=      (self.model_tester.min_size,) * 2
										snake_case_						=      {
										    "pixel_values": torch.randn((2, 3, *size)				,							device=a__							),
										    "mask_labels": torch.randn((2, 10, *size)				,							device=a__							),
										    "class_labels": torch.zeros(2				,							10				,							device=a__							).long(),
										}

										snake_case_						=      MaskFormerForInstanceSegmentation(MaskFormerConfig()							).to(a__							)
										snake_case_						=      model(**a__							)
										self.assertTrue(outputs.loss is not None							)




			def    lowerCAmelCase__				(					self							)				->				Dict:

										'''simple docstring'''




										snake_case_					,      snake_case_						=      self.model_tester.prepare_config_and_inputs_for_common()
										self.model_tester.create_and_check_maskformer_model(a__				,							**a__				,							output_hidden_states=a__							)




			def    lowerCAmelCase__				(					self							)				->				Optional[int]:

										'''simple docstring'''




										snake_case_					,      snake_case_						=      self.model_tester.prepare_config_and_inputs_for_common()

										for model_class in self.all_model_classes:
																	snake_case_						=      model_class(a__							).to(a__							)
																	snake_case_						=      model(**a__				,							output_attentions=a__							)
																	self.assertTrue(outputs.attentions is not None							)




			def    lowerCAmelCase__				(					self							)				->				Dict:

										'''simple docstring'''




										if not self.model_tester.is_training:
																	return
										# only MaskFormerForInstanceSegmentation has the loss
										snake_case_						=      self.all_model_classes[1]
										snake_case_					,      snake_case_					,      snake_case_					,      snake_case_					,      snake_case_						=      self.model_tester.prepare_config_and_inputs()

										snake_case_						=      model_class(a__							)
										model.to(a__							)
										model.train()

										snake_case_						=      model(a__				,							mask_labels=a__				,							class_labels=a__							).loss
										loss.backward()




			def    lowerCAmelCase__				(					self							)				->				Optional[Any]:

										'''simple docstring'''




										snake_case_						=      self.all_model_classes[1]
										snake_case_					,      snake_case_					,      snake_case_					,      snake_case_					,      snake_case_						=      self.model_tester.prepare_config_and_inputs()
										snake_case_						=      True
										snake_case_						=      True

										snake_case_						=      model_class(a__							)
										model.to(a__							)
										model.train()

										snake_case_						=      model(a__				,							mask_labels=a__				,							class_labels=a__							)

										snake_case_						=      outputs.encoder_hidden_states[0]
										encoder_hidden_states.retain_grad()

										snake_case_						=      outputs.pixel_decoder_hidden_states[0]
										pixel_decoder_hidden_states.retain_grad()
										# we requires_grad=True in inputs_embeds (line 2152), the original implementation don't
										snake_case_						=      outputs.transformer_decoder_hidden_states[0]
										transformer_decoder_hidden_states.retain_grad()

										snake_case_						=      outputs.attentions[0]
										attentions.retain_grad()

										outputs.loss.backward(retain_graph=a__							)

										self.assertIsNotNone(encoder_hidden_states.grad							)
										self.assertIsNotNone(pixel_decoder_hidden_states.grad							)
										self.assertIsNotNone(transformer_decoder_hidden_states.grad							)
										self.assertIsNotNone(attentions.grad							)

_SCREAMING_SNAKE_CASE							:   int      		= 1e-4


def      UpperCamelCase_(	):

							'''simple docstring'''






							snake_case_						=      Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"     )
							return image




@require_vision
@slow
class  _snake_case		(	unittest.TestCase							):




			@cached_property
			def    lowerCAmelCase__				(					self							)				->				str:

										'''simple docstring'''




										return (
										    MaskFormerImageProcessor.from_pretrained("facebook/maskformer-swin-small-coco"							)
										    if is_vision_available()
										    else None
										)




			def    lowerCAmelCase__				(					self							)				->				Optional[Any]:

										'''simple docstring'''




										snake_case_						=      MaskFormerModel.from_pretrained("facebook/maskformer-swin-small-coco"							).to(a__							)
										snake_case_						=      self.default_image_processor
										snake_case_						=      prepare_img()
										snake_case_						=      image_processor(a__				,							return_tensors="pt"							).to(a__							)
										snake_case_						=      inputs["pixel_values"].shape
										# check size is divisible by 32
										self.assertTrue((inputs_shape[-1] % 32) == 0 and (inputs_shape[-2] % 32) == 0							)
										# check size
										self.assertEqual(a__				,							(1, 3, 800, 1_088)							)

										with torch.no_grad():
																	snake_case_						=      model(**a__							)

										snake_case_						=      torch.tensor(
										    [[-0.0_4_8_2, 0.9_2_2_8, 0.4_9_5_1], [-0.2_5_4_7, 0.8_0_1_7, 0.8_5_2_7], [-0.0_0_6_9, 0.3_3_8_5, -0.0_0_8_9]]							).to(a__							)
										self.assertTrue(
										    torch.allclose(
										        outputs.encoder_last_hidden_state[0, 0, :3, :3]				,							a__				,							atol=a__							)							)

										snake_case_						=      torch.tensor(
										    [[-0.8_4_2_2, -0.8_4_3_4, -0.9_7_1_8], [-1.0_1_4_4, -0.5_5_6_5, -0.4_1_9_5], [-1.0_0_3_8, -0.4_4_8_4, -0.1_9_6_1]]							).to(a__							)
										self.assertTrue(
										    torch.allclose(
										        outputs.pixel_decoder_last_hidden_state[0, 0, :3, :3]				,							a__				,							atol=a__							)							)

										snake_case_						=      torch.tensor(
										    [[0.2_8_5_2, -0.0_1_5_9, 0.9_7_3_5], [0.6_2_5_4, 0.1_8_5_8, 0.8_5_2_9], [-0.0_6_8_0, -0.4_1_1_6, 1.8_4_1_3]]							).to(a__							)
										self.assertTrue(
										    torch.allclose(
										        outputs.transformer_decoder_last_hidden_state[0, :3, :3]				,							a__				,							atol=a__							)							)




			def    lowerCAmelCase__				(					self							)				->				List[Any]:

										'''simple docstring'''




										snake_case_						=      (
										    MaskFormerForInstanceSegmentation.from_pretrained("facebook/maskformer-swin-small-coco"							)
										    .to(a__							)
										    .eval()
										)
										snake_case_						=      self.default_image_processor
										snake_case_						=      prepare_img()
										snake_case_						=      image_processor(a__				,							return_tensors="pt"							).to(a__							)
										snake_case_						=      inputs["pixel_values"].shape
										# check size is divisible by 32
										self.assertTrue((inputs_shape[-1] % 32) == 0 and (inputs_shape[-2] % 32) == 0							)
										# check size
										self.assertEqual(a__				,							(1, 3, 800, 1_088)							)

										with torch.no_grad():
																	snake_case_						=      model(**a__							)
										# masks_queries_logits
										snake_case_						=      outputs.masks_queries_logits
										self.assertEqual(
										    masks_queries_logits.shape				,							(1, model.config.decoder_config.num_queries, inputs_shape[-2] // 4, inputs_shape[-1] // 4)				,							)
										snake_case_						=      [
										    [-1.3_7_3_7_1_2_4, -1.7_7_2_4_9_3_7, -1.9_3_6_4_2_3_3],
										    [-1.5_9_7_7_2_8_1, -1.9_8_6_7_9_3_9, -2.1_5_2_3_6_9_5],
										    [-1.5_7_9_5_3_9_8, -1.9_2_6_9_8_3_2, -2.0_9_3_9_4_2],
										]
										snake_case_						=      torch.tensor(a__							).to(a__							)
										self.assertTrue(torch.allclose(masks_queries_logits[0, 0, :3, :3]				,							a__				,							atol=a__							)							)
										# class_queries_logits
										snake_case_						=      outputs.class_queries_logits
										self.assertEqual(
										    class_queries_logits.shape				,							(1, model.config.decoder_config.num_queries, model.config.num_labels + 1)							)
										snake_case_						=      torch.tensor(
										    [
										        [1.6512e00, -5.2572e00, -3.3519e00],
										        [3.6169e-02, -5.9025e00, -2.9313e00],
										        [1.0766e-04, -7.7630e00, -5.1263e00],
										    ]							).to(a__							)
										self.assertTrue(torch.allclose(outputs.class_queries_logits[0, :3, :3]				,							a__				,							atol=a__							)							)




			def    lowerCAmelCase__				(					self							)				->				Union[str, Any]:

										'''simple docstring'''




										snake_case_						=      (
										    MaskFormerForInstanceSegmentation.from_pretrained("facebook/maskformer-resnet101-coco-stuff"							)
										    .to(a__							)
										    .eval()
										)
										snake_case_						=      self.default_image_processor
										snake_case_						=      prepare_img()
										snake_case_						=      image_processor(a__				,							return_tensors="pt"							).to(a__							)
										snake_case_						=      inputs["pixel_values"].shape
										# check size is divisible by 32
										self.assertTrue((inputs_shape[-1] % 32) == 0 and (inputs_shape[-2] % 32) == 0							)
										# check size
										self.assertEqual(a__				,							(1, 3, 800, 1_088)							)

										with torch.no_grad():
																	snake_case_						=      model(**a__							)
										# masks_queries_logits
										snake_case_						=      outputs.masks_queries_logits
										self.assertEqual(
										    masks_queries_logits.shape				,							(1, model.config.decoder_config.num_queries, inputs_shape[-2] // 4, inputs_shape[-1] // 4)				,							)
										snake_case_						=      [[-0.9_0_4_6, -2.6_3_6_6, -4.6_0_6_2], [-3.4_1_7_9, -5.7_8_9_0, -8.8_0_5_7], [-4.9_1_7_9, -7.6_5_6_0, -1_0.7_7_1_1]]
										snake_case_						=      torch.tensor(a__							).to(a__							)
										self.assertTrue(torch.allclose(masks_queries_logits[0, 0, :3, :3]				,							a__				,							atol=a__							)							)
										# class_queries_logits
										snake_case_						=      outputs.class_queries_logits
										self.assertEqual(
										    class_queries_logits.shape				,							(1, model.config.decoder_config.num_queries, model.config.num_labels + 1)							)
										snake_case_						=      torch.tensor(
										    [[4.7_1_8_8, -3.2_5_8_5, -2.8_8_5_7], [6.6_8_7_1, -2.9_1_8_1, -1.2_4_8_7], [7.2_4_4_9, -2.2_7_6_4, -2.1_8_7_4]]							).to(a__							)
										self.assertTrue(torch.allclose(outputs.class_queries_logits[0, :3, :3]				,							a__				,							atol=a__							)							)




			def    lowerCAmelCase__				(					self							)				->				Union[str, Any]:

										'''simple docstring'''




										snake_case_						=      (
										    MaskFormerForInstanceSegmentation.from_pretrained("facebook/maskformer-swin-small-coco"							)
										    .to(a__							)
										    .eval()
										)
										snake_case_						=      self.default_image_processor

										snake_case_						=      image_processor(
										    [np.zeros((3, 800, 1_333)							), np.zeros((3, 800, 1_333)							)]				,							segmentation_maps=[np.zeros((384, 384)							).astype(np.floataa							), np.zeros((384, 384)							).astype(np.floataa							)]				,							return_tensors="pt"				,							)

										snake_case_						=      inputs["pixel_values"].to(a__							)
										snake_case_						=      [el.to(a__							) for el in inputs["mask_labels"]]
										snake_case_						=      [el.to(a__							) for el in inputs["class_labels"]]

										with torch.no_grad():
																	snake_case_						=      model(**a__							)

										self.assertTrue(outputs.loss is not None							)








import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
    from .tokenization_mbart import MBartTokenizer
else:
    lowerCAmelCase__      		= None


lowerCAmelCase__      		= logging.get_logger(__name__)


lowerCAmelCase__      		= {'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

lowerCAmelCase__      		= {
    'vocab_file': {
        'facebook/mbart-large-en-ro': (
            'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'
        ),
        'facebook/mbart-large-cc25': (
            'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/mbart-large-en-ro': 'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json',
        'facebook/mbart-large-cc25': 'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json',
    },
}

lowerCAmelCase__      		= {
    'facebook/mbart-large-en-ro': 10_24,
    'facebook/mbart-large-cc25': 10_24,
}

# fmt: off
lowerCAmelCase__      		= ['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN']







class      lowerCAmelCase__       (				a):




     '''simple docstring'''
     __SCREAMING_SNAKE_CASE  			= VOCAB_FILES_NAMES
     __SCREAMING_SNAKE_CASE  			= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __SCREAMING_SNAKE_CASE  			= PRETRAINED_VOCAB_FILES_MAP
     __SCREAMING_SNAKE_CASE  			= ["input_ids", "attention_mask"]
     __SCREAMING_SNAKE_CASE  			= MBartTokenizer

     __SCREAMING_SNAKE_CASE  			= []
     __SCREAMING_SNAKE_CASE  			= []


     def __init__(						self ,						__lowerCamelCase=None ,						__lowerCamelCase=None ,						__lowerCamelCase="<s>" ,						__lowerCamelCase="</s>" ,						__lowerCamelCase="</s>" ,						__lowerCamelCase="<s>" ,						__lowerCamelCase="<unk>" ,						__lowerCamelCase="<pad>" ,						__lowerCamelCase="<mask>" ,						__lowerCamelCase=None ,						__lowerCamelCase=None ,						__lowerCamelCase=None ,						**__lowerCamelCase ,						)			->				Optional[int]:
            # Mask token behave like a normal word, i.e. include the space before it
            _A      :       List[str]				    = AddedToken(__lowerCamelCase ,						lstrip=__lowerCamelCase ,						rstrip=__lowerCamelCase) if isinstance(__lowerCamelCase ,						__lowerCamelCase) else mask_token

            super().__init__(
                vocab_file=__lowerCamelCase ,						tokenizer_file=__lowerCamelCase ,						bos_token=__lowerCamelCase ,						eos_token=__lowerCamelCase ,						sep_token=__lowerCamelCase ,						cls_token=__lowerCamelCase ,						unk_token=__lowerCamelCase ,						pad_token=__lowerCamelCase ,						mask_token=__lowerCamelCase ,						src_lang=__lowerCamelCase ,						tgt_lang=__lowerCamelCase ,						additional_special_tokens=__lowerCamelCase ,						**__lowerCamelCase ,						)

            _A      :       Union[str, Any]				    = vocab_file
            _A      :       int				    = False if not self.vocab_file else True

            _A      :       Optional[int]				    = FAIRSEQ_LANGUAGE_CODES.copy()

            if additional_special_tokens is not None:
                   # Only add those special tokens if they are not already there.
                   _additional_special_tokens.extend(
                       [t for t in additional_special_tokens if t not in _additional_special_tokens])

            self.add_special_tokens({"additional_special_tokens": _additional_special_tokens})
            _A      :       Union[str, Any]				    = {
                lang_code: self.convert_tokens_to_ids(__lowerCamelCase) for lang_code in FAIRSEQ_LANGUAGE_CODES
            }

            _A      :       Optional[int]				    = src_lang if src_lang is not None else "en_XX"
            _A      :       Union[str, Any]				    = self.convert_tokens_to_ids(self._src_lang)
            _A      :       int				    = tgt_lang
            self.set_src_lang_special_tokens(self._src_lang)


     @property
     def  _lowerCamelCase (						self)			->				str:
            return self._src_lang


     @src_lang.setter
     def  _lowerCamelCase (						self ,						__lowerCamelCase)			->				None:
            _A      :       Dict				    = new_src_lang
            self.set_src_lang_special_tokens(self._src_lang)


     def  _lowerCamelCase (						self ,						__lowerCamelCase ,						__lowerCamelCase = None)			->				List[int]:
            if token_ids_a is None:
                   return self.prefix_tokens + token_ids_a + self.suffix_tokens
            # We don't expect to process pairs, but leave the pair logic for API consistency
            return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens


     def  _lowerCamelCase (						self ,						__lowerCamelCase ,						__lowerCamelCase = None)			->				List[int]:
            _A      :       List[str]				    = [self.sep_token_id]
            _A      :       List[str]				    = [self.cls_token_id]

            if token_ids_a is None:
                   return len(cls + token_ids_a + sep) * [0]
            return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]


     def  _lowerCamelCase (						self ,						__lowerCamelCase ,						__lowerCamelCase ,						__lowerCamelCase ,						__lowerCamelCase ,						**__lowerCamelCase)			->				Dict:
            if src_lang is None or tgt_lang is None:
                   raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model")
            _A      :       str				    = src_lang
            _A      :       Any				    = self(__lowerCamelCase ,						add_special_tokens=__lowerCamelCase ,						return_tensors=__lowerCamelCase ,						**__lowerCamelCase)
            _A      :       Tuple				    = self.convert_tokens_to_ids(__lowerCamelCase)
            _A      :       Dict				    = tgt_lang_id
            return inputs


     def  _lowerCamelCase (						self ,						__lowerCamelCase ,						__lowerCamelCase = "en_XX" ,						__lowerCamelCase = None ,						__lowerCamelCase = "ro_RO" ,						**__lowerCamelCase ,						)			->				BatchEncoding:
            _A      :       Any				    = src_lang
            _A      :       int				    = tgt_lang
            return super().prepare_seqaseq_batch(__lowerCamelCase ,						__lowerCamelCase ,						**__lowerCamelCase)


     def  _lowerCamelCase (						self)			->				List[str]:
            return self.set_src_lang_special_tokens(self.src_lang)


     def  _lowerCamelCase (						self)			->				List[Any]:
            return self.set_tgt_lang_special_tokens(self.tgt_lang)


     def  _lowerCamelCase (						self ,						__lowerCamelCase)			->				None:
            _A      :       int				    = self.convert_tokens_to_ids(__lowerCamelCase)
            _A      :       int				    = []
            _A      :       List[str]				    = [self.eos_token_id, self.cur_lang_code]

            _A      :       Union[str, Any]				    = self.convert_ids_to_tokens(self.prefix_tokens)
            _A      :       str				    = self.convert_ids_to_tokens(self.suffix_tokens)

            _A      :       List[Any]				    = processors.TemplateProcessing(
                single=prefix_tokens_str + ["$A"] + suffix_tokens_str ,						pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str ,						special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str ,						self.prefix_tokens + self.suffix_tokens)) ,						)


     def  _lowerCamelCase (						self ,						__lowerCamelCase)			->				None:
            _A      :       Optional[int]				    = self.convert_tokens_to_ids(__lowerCamelCase)
            _A      :       List[Any]				    = []
            _A      :       str				    = [self.eos_token_id, self.cur_lang_code]

            _A      :       Optional[int]				    = self.convert_ids_to_tokens(self.prefix_tokens)
            _A      :       int				    = self.convert_ids_to_tokens(self.suffix_tokens)

            _A      :       str				    = processors.TemplateProcessing(
                single=prefix_tokens_str + ["$A"] + suffix_tokens_str ,						pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str ,						special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str ,						self.prefix_tokens + self.suffix_tokens)) ,						)


     def  _lowerCamelCase (						self ,						__lowerCamelCase ,						__lowerCamelCase = None)			->				Tuple[str]:
            if not self.can_save_slow_tokenizer:
                   raise ValueError(
                       "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                       "tokenizer.")

            if not os.path.isdir(__lowerCamelCase):
                   logger.error(F"Vocabulary path ({save_directory}) should be a directory.")
                   return
            _A      :       int				    = os.path.join(
                __lowerCamelCase ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])

            if os.path.abspath(self.vocab_file) != os.path.abspath(__lowerCamelCase):
                   copyfile(self.vocab_file ,						__lowerCamelCase)

            return (out_vocab_file,)
