
import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
						from .tokenization_mbart import MBartTokenizer
else:
						__a  			=					None


__a  			=					logging.get_logger(__name__)


__a  			=					{'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

__a  			=					{
    'vocab_file': {
        'facebook/mbart-large-en-ro': (
            'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'
        ),
        'facebook/mbart-large-cc25': (
            'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/mbart-large-en-ro': 'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json',
        'facebook/mbart-large-cc25': 'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json',
    },
}

__a  			=					{
    'facebook/mbart-large-en-ro': 1_0_2_4,
    'facebook/mbart-large-cc25': 1_0_2_4,
}

# fmt: off
__a  			=					['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN']
class  lowercase__(				_UpperCAmelCase							):



	"""simple docstring"""




	a			:Dict         =	VOCAB_FILES_NAMES
	a			:Optional[int]         =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a			:str         =	PRETRAINED_VOCAB_FILES_MAP
	a			:int         =	['input_ids', 'attention_mask']
	a			:Tuple         =	MBartTokenizer

	a			:List[int]         =	[]
	a			:List[int]         =	[]
	def __init__(       self  :			Optional[int]       ,							SCREAMING_SNAKE_CASE_  :			str=None       ,							SCREAMING_SNAKE_CASE_  :			int=None       ,							SCREAMING_SNAKE_CASE_  :			Any="<s>"       ,							SCREAMING_SNAKE_CASE_  :			Any="</s>"       ,							SCREAMING_SNAKE_CASE_  :			Any="</s>"       ,							SCREAMING_SNAKE_CASE_  :			List[str]="<s>"       ,							SCREAMING_SNAKE_CASE_  :			List[str]="<unk>"       ,							SCREAMING_SNAKE_CASE_  :			Dict="<pad>"       ,							SCREAMING_SNAKE_CASE_  :			List[Any]="<mask>"       ,							SCREAMING_SNAKE_CASE_  :			Union[str, Any]=None       ,							SCREAMING_SNAKE_CASE_  :			Any=None       ,							SCREAMING_SNAKE_CASE_  :			Union[str, Any]=None       ,							**SCREAMING_SNAKE_CASE_  :			int       ,							)       ->     str:
					lowercase_       =					AddedToken(a__       ,							lstrip=a__       ,							rstrip=a__       ) if isinstance(a__       ,							a__       ) else mask_token

					super().__init__(
					    vocab_file=a__       ,							tokenizer_file=a__       ,							bos_token=a__       ,							eos_token=a__       ,							sep_token=a__       ,							cls_token=a__       ,							unk_token=a__       ,							pad_token=a__       ,							mask_token=a__       ,							src_lang=a__       ,							tgt_lang=a__       ,							additional_special_tokens=a__       ,							**a__       ,							)

					lowercase_       =					vocab_file
					lowercase_       =					False if not self.vocab_file else True

					lowercase_       =					FAIRSEQ_LANGUAGE_CODES.copy()

					if additional_special_tokens is not None:
									# Only add those special tokens if they are not already there.
									_additional_special_tokens.extend(
									    [t for t in additional_special_tokens if t not in _additional_special_tokens]       )

					self.add_special_tokens({'''additional_special_tokens''': _additional_special_tokens}       )
					lowercase_       =					{
					    lang_code: self.convert_tokens_to_ids(a__       ) for lang_code in FAIRSEQ_LANGUAGE_CODES
					}

					lowercase_       =					src_lang if src_lang is not None else '''en_XX'''
					lowercase_       =					self.convert_tokens_to_ids(self._src_lang       )
					lowercase_       =					tgt_lang
					self.set_src_lang_special_tokens(self._src_lang       )
	@property
	def 						_lowercase  (       self  :			Optional[int]       )       ->     Dict:
					return self._src_lang
	@src_lang.setter
	def 						_lowercase  (       self  :			Tuple       ,							SCREAMING_SNAKE_CASE_  :			str       )       ->     List[Any]:
					lowercase_       =					new_src_lang
					self.set_src_lang_special_tokens(self._src_lang       )
	def 						_lowercase  (       self  :			Tuple       ,							SCREAMING_SNAKE_CASE_  :			List[int]       ,							SCREAMING_SNAKE_CASE_  :			Optional[List[int]] = None       )       ->     Tuple:



					if token_ids_a is None:
									return self.prefix_tokens + token_ids_a + self.suffix_tokens
					# We don't expect to process pairs, but leave the pair logic for API consistency
					return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens
	def 						_lowercase  (       self  :			List[Any]       ,							SCREAMING_SNAKE_CASE_  :			List[int]       ,							SCREAMING_SNAKE_CASE_  :			Optional[List[int]] = None       )       ->     Tuple:
					lowercase_       =					[self.sep_token_id]
					lowercase_       =					[self.cls_token_id]

					if token_ids_a is None:
									return len(cls + token_ids_a + sep       ) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]
	def 						_lowercase  (       self  :			List[Any]       ,							SCREAMING_SNAKE_CASE_  :			Optional[Any]       ,							SCREAMING_SNAKE_CASE_  :			str       ,							SCREAMING_SNAKE_CASE_  :			Optional[str]       ,							SCREAMING_SNAKE_CASE_  :			Optional[str]       ,							**SCREAMING_SNAKE_CASE_  :			int       )       ->     Dict:



					if src_lang is None or tgt_lang is None:
									raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''       )
					lowercase_       =					src_lang
					lowercase_       =					self(a__       ,							add_special_tokens=a__       ,							return_tensors=a__       ,							**a__       )
					lowercase_       =					self.convert_tokens_to_ids(a__       )
					lowercase_       =					tgt_lang_id
					return inputs
	def 						_lowercase  (       self  :			Tuple       ,							SCREAMING_SNAKE_CASE_  :			List[str]       ,							SCREAMING_SNAKE_CASE_  :			str = "en_XX"       ,							SCREAMING_SNAKE_CASE_  :			Optional[List[str]] = None       ,							SCREAMING_SNAKE_CASE_  :			str = "ro_RO"       ,							**SCREAMING_SNAKE_CASE_  :			Tuple       ,							)       ->     Dict:
					lowercase_       =					src_lang
					lowercase_       =					tgt_lang
					return super().prepare_seqaseq_batch(a__       ,							a__       ,							**a__       )
	def 						_lowercase  (       self  :			str       )       ->     Optional[int]:
					return self.set_src_lang_special_tokens(self.src_lang       )
	def 						_lowercase  (       self  :			List[Any]       )       ->     Any:
					return self.set_tgt_lang_special_tokens(self.tgt_lang       )
	def 						_lowercase  (       self  :			Optional[int]       ,							SCREAMING_SNAKE_CASE_  :			str       )       ->     Tuple:
					lowercase_       =					self.convert_tokens_to_ids(a__       )
					lowercase_       =					[]
					lowercase_       =					[self.eos_token_id, self.cur_lang_code]

					lowercase_       =					self.convert_ids_to_tokens(self.prefix_tokens       )
					lowercase_       =					self.convert_ids_to_tokens(self.suffix_tokens       )

					lowercase_       =					processors.TemplateProcessing(
					    single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str       ,							pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str       ,							special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str       ,							self.prefix_tokens + self.suffix_tokens       )       )       ,							)
	def 						_lowercase  (       self  :			Union[str, Any]       ,							SCREAMING_SNAKE_CASE_  :			str       )       ->     Any:
					lowercase_       =					self.convert_tokens_to_ids(a__       )
					lowercase_       =					[]
					lowercase_       =					[self.eos_token_id, self.cur_lang_code]

					lowercase_       =					self.convert_ids_to_tokens(self.prefix_tokens       )
					lowercase_       =					self.convert_ids_to_tokens(self.suffix_tokens       )

					lowercase_       =					processors.TemplateProcessing(
					    single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str       ,							pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str       ,							special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str       ,							self.prefix_tokens + self.suffix_tokens       )       )       ,							)




	def 						_lowercase  (       self  :			int       ,							SCREAMING_SNAKE_CASE_  :			str       ,							SCREAMING_SNAKE_CASE_  :			Optional[str] = None       )       ->     str:



					if not self.can_save_slow_tokenizer:
									raise ValueError(
									    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
									    '''tokenizer.'''       )

					if not os.path.isdir(a__       ):
									logger.error(f'''Vocabulary path ({save_directory}) should be a directory.'''       )
									return
					lowercase_       =					os.path.join(
					    a__       ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )

					if os.path.abspath(self.vocab_file       ) != os.path.abspath(a__       ):
									copyfile(self.vocab_file       ,							a__       )

					return (out_vocab_file,)











import inspect
import unittest

from transformers import ConvNextConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import ConvNextBackbone, ConvNextForImageClassification, ConvNextModel
    from transformers.models.convnext.modeling_convnext import CONVNEXT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
    from PIL import Image

    from transformers import AutoImageProcessor

class 		SCREAMING_SNAKE_CASE__				:


      def __init__(self       :      Any				,			a__       :      List[Any]				,			a__       :      Dict=13				,			a__       :      str=32				,			a__       :      Tuple=3				,			a__       :      Optional[Any]=4				,			a__       :      Optional[int]=[10, 20, 30, 40]				,			a__       :      List[Any]=[2, 2, 3, 2]				,			a__       :      List[Any]=True				,			a__       :      int=True				,			a__       :      List[Any]=37				,			a__       :      Any="gelu"				,			a__       :      int=10				,			a__       :      Dict=0.0_2				,			a__       :      Dict=["stage2", "stage3", "stage4"]				,			a__       :      Tuple=[2, 3, 4]				,			a__       :      List[str]=None				,			):





         """simple docstring"""



         __snake_case							       =   parent
         __snake_case							       =   batch_size
         __snake_case							       =   image_size
         __snake_case							       =   num_channels
         __snake_case							       =   num_stages
         __snake_case							       =   hidden_sizes
         __snake_case							       =   depths
         __snake_case							       =   is_training
         __snake_case							       =   use_labels
         __snake_case							       =   intermediate_size
         __snake_case							       =   hidden_act
         __snake_case							       =   num_labels
         __snake_case							       =   initializer_range
         __snake_case							       =   out_features
         __snake_case							       =   out_indices
         __snake_case							       =   scope


      def    a (self       :      Dict      ):





         """simple docstring"""



         __snake_case							       =   floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]      )

         __snake_case							       =   None
         if self.use_labels:
            __snake_case							       =   ids_tensor([self.batch_size]				,			self.num_labels      )

         __snake_case							       =   self.get_config()
         return config, pixel_values, labels


      def    a (self       :      List[str]      ):





         """simple docstring"""



         return ConvNextConfig(
             num_channels=self.num_channels				,			hidden_sizes=self.hidden_sizes				,			depths=self.depths				,			num_stages=self.num_stages				,			hidden_act=self.hidden_act				,			is_decoder=a__				,			initializer_range=self.initializer_range				,			out_features=self.out_features				,			out_indices=self.out_indices				,			num_labels=self.num_labels				,			)


      def    a (self       :      str				,			a__       :      Union[str, Any]				,			a__       :      List[str]				,			a__       :      List[Any]      ):





         """simple docstring"""



         __snake_case							       =   ConvNextModel(config=a__      )
         model.to(a__      )
         model.eval()
         __snake_case							       =   model(a__      )
         # expected last hidden states: B, C, H // 32, W // 32
         self.parent.assertEqual(
             result.last_hidden_state.shape				,			(self.batch_size, self.hidden_sizes[-1], self.image_size // 32, self.image_size // 32)				,			)


      def    a (self       :      Optional[Any]				,			a__       :      List[Any]				,			a__       :      str				,			a__       :      List[Any]      ):





         """simple docstring"""



         __snake_case							       =   ConvNextForImageClassification(a__      )
         model.to(a__      )
         model.eval()
         __snake_case							       =   model(a__				,			labels=a__      )
         self.parent.assertEqual(result.logits.shape				,			(self.batch_size, self.num_labels)      )


      def    a (self       :      Tuple				,			a__       :      List[Any]				,			a__       :      List[str]				,			a__       :      List[str]      ):





         """simple docstring"""



         __snake_case							       =   ConvNextBackbone(config=a__      )
         model.to(a__      )
         model.eval()
         __snake_case							       =   model(a__      )

         # verify hidden states
         self.parent.assertEqual(len(result.feature_maps      )				,			len(config.out_features      )      )
         self.parent.assertListEqual(list(result.feature_maps[0].shape      )				,			[self.batch_size, self.hidden_sizes[1], 4, 4]      )

         # verify channels
         self.parent.assertEqual(len(model.channels      )				,			len(config.out_features      )      )
         self.parent.assertListEqual(model.channels				,			config.hidden_sizes[1:]      )

         # verify backbone works with out_features=None
         __snake_case							       =   None
         __snake_case							       =   ConvNextBackbone(config=a__      )
         model.to(a__      )
         model.eval()
         __snake_case							       =   model(a__      )

         # verify feature maps
         self.parent.assertEqual(len(result.feature_maps      )				,			1      )
         self.parent.assertListEqual(list(result.feature_maps[0].shape      )				,			[self.batch_size, self.hidden_sizes[-1], 1, 1]      )

         # verify channels
         self.parent.assertEqual(len(model.channels      )				,			1      )
         self.parent.assertListEqual(model.channels				,			[config.hidden_sizes[-1]]      )


      def    a (self       :      Tuple      ):





         """simple docstring"""



         __snake_case							       =   self.prepare_config_and_inputs()
         __snake_case       ,	__snake_case       ,	__snake_case							       =   config_and_inputs
         __snake_case							       =   {'''pixel_values''': pixel_values}
         return config, inputs_dict






@require_torch
class 		SCREAMING_SNAKE_CASE__				(       _UpperCAmelCase						,      _UpperCAmelCase						,      unittest.TestCase  ):
      A_						:				Dict							=    (
          (
              ConvNextModel,
              ConvNextForImageClassification,
              ConvNextBackbone,
          )
          if is_torch_available()
          else ()
      )
      A_						:				Optional[Any]							=    (
          {'feature-extraction': ConvNextModel, 'image-classification': ConvNextForImageClassification}
          if is_torch_available()
          else {}
      )

      A_						:				Dict							=    True
      A_						:				Optional[Any]							=    False
      A_						:				int							=    False
      A_						:				int							=    False
      A_						:				List[str]							=    False


      def    a (self       :      List[str]      ):





         """simple docstring"""



         __snake_case							       =   ConvNextModelTester(self      )
         __snake_case							       =   ConfigTester(self				,			config_class=a__				,			has_text_modality=a__				,			hidden_size=37      )


      def    a (self       :      Tuple      ):





         """simple docstring"""



         self.create_and_test_config_common_properties()
         self.config_tester.create_and_test_config_to_json_string()
         self.config_tester.create_and_test_config_to_json_file()
         self.config_tester.create_and_test_config_from_and_save_pretrained()
         self.config_tester.create_and_test_config_with_num_labels()
         self.config_tester.check_config_can_be_init_without_params()
         self.config_tester.check_config_arguments_init()


      def    a (self       :      str      ):





         """simple docstring"""



         return


      @unittest.skip(reason='''ConvNext does not use inputs_embeds'''      )
      def    a (self       :      int      ):





         """simple docstring"""



         pass


      @unittest.skip(reason='''ConvNext does not support input and output embeddings'''      )
      def    a (self       :      Dict      ):





         """simple docstring"""



         pass


      @unittest.skip(reason='''ConvNext does not use feedforward chunking'''      )
      def    a (self       :      List[Any]      ):





         """simple docstring"""



         pass


      def    a (self       :      Optional[Any]      ):





         """simple docstring"""



         __snake_case       ,	__snake_case							       =   self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
            __snake_case							       =   model_class(a__      )
            __snake_case							       =   inspect.signature(model.forward      )
            # signature.parameters is an OrderedDict => so arg_names order is deterministic
            __snake_case							       =   [*signature.parameters.keys()]

            __snake_case							       =   ['''pixel_values''']
            self.assertListEqual(arg_names[:1]				,			a__      )


      def    a (self       :      List[Any]      ):





         """simple docstring"""



         __snake_case							       =   self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*a__      )


      def    a (self       :      Dict      ):





         """simple docstring"""



         __snake_case							       =   self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_backbone(*a__      )


      def    a (self       :      Dict      ):





         """simple docstring"""



         def check_hidden_states_output(a__       :      List[str]				,			a__       :      str				,			a__       :      Tuple      ):
            __snake_case							       =   model_class(a__      )
            model.to(a__      )
            model.eval()

            with torch.no_grad():
               __snake_case							       =   model(**self._prepare_for_class(a__				,			a__      )      )

            __snake_case							       =   outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

            __snake_case							       =   self.model_tester.num_stages
            self.assertEqual(len(a__      )				,			expected_num_stages + 1      )

            # ConvNext's feature maps are of shape (batch_size, num_channels, height, width)
            self.assertListEqual(
                list(hidden_states[0].shape[-2:]      )				,			[self.model_tester.image_size // 4, self.model_tester.image_size // 4]				,			)

         __snake_case       ,	__snake_case							       =   self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
            __snake_case							       =   True
            check_hidden_states_output(a__				,			a__				,			a__      )

            # check that output_hidden_states also work using config
            del inputs_dict["output_hidden_states"]
            __snake_case							       =   True

            check_hidden_states_output(a__				,			a__				,			a__      )


      def    a (self       :      Optional[Any]      ):





         """simple docstring"""



         __snake_case							       =   self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_image_classification(*a__      )


      @slow
      def    a (self       :      Any      ):





         """simple docstring"""



         for model_name in CONVNEXT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            __snake_case							       =   ConvNextModel.from_pretrained(a__      )
            self.assertIsNotNone(a__      )






def lowerCamelCase__    (							)      ->				List[str]:
   __snake_case							       =   Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''       )
   return image

@require_torch
@require_vision
class 		SCREAMING_SNAKE_CASE__				(       unittest.TestCase  ):


      @cached_property
      def    a (self       :      Tuple      ):





         """simple docstring"""



         return AutoImageProcessor.from_pretrained('''facebook/convnext-tiny-224'''      ) if is_vision_available() else None


      @slow
      def    a (self       :      Optional[Any]      ):





         """simple docstring"""



         __snake_case							       =   ConvNextForImageClassification.from_pretrained('''facebook/convnext-tiny-224'''      ).to(a__      )

         __snake_case							       =   self.default_image_processor
         __snake_case							       =   prepare_img()
         __snake_case							       =   image_processor(images=a__				,			return_tensors='''pt'''      ).to(a__      )

         # forward pass
         with torch.no_grad():
            __snake_case							       =   model(**a__      )

         # verify the logits
         __snake_case							       =   torch.Size((1, 1000)      )
         self.assertEqual(outputs.logits.shape				,			a__      )

         __snake_case							       =   torch.tensor([-0.0_2_6_0, -0.4_7_3_9, 0.1_9_1_1]      ).to(a__      )

         self.assertTrue(torch.allclose(outputs.logits[0, :3]				,			a__				,			atol=1E-4      )      )






@require_torch
class 		SCREAMING_SNAKE_CASE__				(       unittest.TestCase						,      _UpperCAmelCase  ):
      A_						:				Union[str, Any]							=    (ConvNextBackbone,) if is_torch_available() else ()
      A_						:				List[Any]							=    ConvNextConfig

      A_						:				Optional[Any]							=    False


      def    a (self       :      Optional[int]      ):





         """simple docstring"""



         __snake_case							       =   ConvNextModelTester(self      )

