


import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartTokenizer, MBartTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCAmelCase	: List[str]  				=get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
						from transformers.models.mbart.modeling_mbart import shift_tokens_right

__lowerCAmelCase	: str  				=2_5_0_0_0_4
__lowerCAmelCase	: Any  				=2_5_0_0_2_0
@require_sentencepiece
@require_tokenizers
class        _lowercase      (			A__						,			unittest.TestCase       ):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE__				:			List[str]						=		MBartTokenizer
				SCREAMING_SNAKE_CASE__				:			str						=		MBartTokenizerFast
				SCREAMING_SNAKE_CASE__				:			List[Any]						=		True
				SCREAMING_SNAKE_CASE__				:			Optional[int]						=		True


				def 		__magic_name__(       self			:List[str]	)		->			str:
										super().setUp()

										# We have a SentencePiece fixture for testing
										__SCREAMING_SNAKE_CASE						:     Any					=		MBartTokenizer(lowerCAmelCase__			,			keep_accents=lowerCAmelCase__	)
										tokenizer.save_pretrained(self.tmpdirname	)


				def 		__magic_name__(       self			:Any	)		->			Tuple:
										__SCREAMING_SNAKE_CASE						:     Dict					=		MBartTokenizer(lowerCAmelCase__			,			keep_accents=lowerCAmelCase__	)

										__SCREAMING_SNAKE_CASE						:     Dict					=		tokenizer.tokenize('''This is a test'''	)
										self.assertListEqual(lowerCAmelCase__			,			['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']	)

										self.assertListEqual(
										    tokenizer.convert_tokens_to_ids(lowerCAmelCase__	)			,			[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]			,			)

										__SCREAMING_SNAKE_CASE						:     Any					=		tokenizer.tokenize('''I was born in 92000, and this is falsé.'''	)
										self.assertListEqual(
										    lowerCAmelCase__			,			[
										        SPIECE_UNDERLINE + '''I''',
										        SPIECE_UNDERLINE + '''was''',
										        SPIECE_UNDERLINE + '''b''',
										        '''or''',
										        '''n''',
										        SPIECE_UNDERLINE + '''in''',
										        SPIECE_UNDERLINE + '''''',
										        '''9''',
										        '''2''',
										        '''0''',
										        '''0''',
										        '''0''',
										        ''',''',
										        SPIECE_UNDERLINE + '''and''',
										        SPIECE_UNDERLINE + '''this''',
										        SPIECE_UNDERLINE + '''is''',
										        SPIECE_UNDERLINE + '''f''',
										        '''al''',
										        '''s''',
										        '''é''',
										        '''.''',
										    ]			,			)
										__SCREAMING_SNAKE_CASE						:     Any					=		tokenizer.convert_tokens_to_ids(lowerCAmelCase__	)
										self.assertListEqual(
										    lowerCAmelCase__			,			[
										        value + tokenizer.fairseq_offset
										        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
										        #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
										    ]			,			)

										__SCREAMING_SNAKE_CASE						:     Tuple					=		tokenizer.convert_ids_to_tokens(lowerCAmelCase__	)
										self.assertListEqual(
										    lowerCAmelCase__			,			[
										        SPIECE_UNDERLINE + '''I''',
										        SPIECE_UNDERLINE + '''was''',
										        SPIECE_UNDERLINE + '''b''',
										        '''or''',
										        '''n''',
										        SPIECE_UNDERLINE + '''in''',
										        SPIECE_UNDERLINE + '''''',
										        '''<unk>''',
										        '''2''',
										        '''0''',
										        '''0''',
										        '''0''',
										        ''',''',
										        SPIECE_UNDERLINE + '''and''',
										        SPIECE_UNDERLINE + '''this''',
										        SPIECE_UNDERLINE + '''is''',
										        SPIECE_UNDERLINE + '''f''',
										        '''al''',
										        '''s''',
										        '''<unk>''',
										        '''.''',
										    ]			,			)



				def 		__magic_name__(       self			:Dict	)		->			Union[str, Any]:
										if not self.test_slow_tokenizer:
																# as we don't have a slow version, we can't compare the outputs between slow and fast versions
																return

										__SCREAMING_SNAKE_CASE						:     str					=		(self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-mbart''', {})
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
																						__SCREAMING_SNAKE_CASE						:     Optional[int]					=		self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__			,			**lowerCAmelCase__	)
																						__SCREAMING_SNAKE_CASE						:     List[Any]					=		self.tokenizer_class.from_pretrained(lowerCAmelCase__			,			**lowerCAmelCase__	)

																						__SCREAMING_SNAKE_CASE						:     Dict					=		tempfile.mkdtemp()

																						__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		tokenizer_r.save_pretrained(lowerCAmelCase__	)
																						__SCREAMING_SNAKE_CASE						:     List[Any]					=		tokenizer_p.save_pretrained(lowerCAmelCase__	)

																						# Checks it save with the same files + the tokenizer.json file for the fast one
																						self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files	)	)
																						__SCREAMING_SNAKE_CASE						:     Dict					=		tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f	)
																						self.assertSequenceEqual(lowerCAmelCase__			,			lowerCAmelCase__	)

																						# Checks everything loads correctly in the same way
																						__SCREAMING_SNAKE_CASE						:     str					=		tokenizer_r.from_pretrained(lowerCAmelCase__	)
																						__SCREAMING_SNAKE_CASE						:     int					=		tokenizer_p.from_pretrained(lowerCAmelCase__	)

																						# Check special tokens are set accordingly on Rust and Python
																						for key in tokenizer_pp.special_tokens_map:
																												self.assertTrue(hasattr(lowerCAmelCase__			,			lowerCAmelCase__	)	)
																												# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
																												# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

																						shutil.rmtree(lowerCAmelCase__	)

																						# Save tokenizer rust, legacy_format=True
																						__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		tempfile.mkdtemp()

																						__SCREAMING_SNAKE_CASE						:     List[str]					=		tokenizer_r.save_pretrained(lowerCAmelCase__			,			legacy_format=lowerCAmelCase__	)
																						__SCREAMING_SNAKE_CASE						:     List[str]					=		tokenizer_p.save_pretrained(lowerCAmelCase__	)

																						# Checks it save with the same files
																						self.assertSequenceEqual(lowerCAmelCase__			,			lowerCAmelCase__	)

																						# Checks everything loads correctly in the same way
																						__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		tokenizer_r.from_pretrained(lowerCAmelCase__	)
																						__SCREAMING_SNAKE_CASE						:     Tuple					=		tokenizer_p.from_pretrained(lowerCAmelCase__	)

																						# Check special tokens are set accordingly on Rust and Python
																						for key in tokenizer_pp.special_tokens_map:
																												self.assertTrue(hasattr(lowerCAmelCase__			,			lowerCAmelCase__	)	)

																						shutil.rmtree(lowerCAmelCase__	)

																						# Save tokenizer rust, legacy_format=False
																						__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		tempfile.mkdtemp()

																						__SCREAMING_SNAKE_CASE						:     Dict					=		tokenizer_r.save_pretrained(lowerCAmelCase__			,			legacy_format=lowerCAmelCase__	)
																						__SCREAMING_SNAKE_CASE						:     int					=		tokenizer_p.save_pretrained(lowerCAmelCase__	)

																						# Checks it saved the tokenizer.json file
																						self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files	)	)

																						# Checks everything loads correctly in the same way
																						__SCREAMING_SNAKE_CASE						:     Dict					=		tokenizer_r.from_pretrained(lowerCAmelCase__	)
																						__SCREAMING_SNAKE_CASE						:     List[Any]					=		tokenizer_p.from_pretrained(lowerCAmelCase__	)

																						# Check special tokens are set accordingly on Rust and Python
																						for key in tokenizer_pp.special_tokens_map:
																												self.assertTrue(hasattr(lowerCAmelCase__			,			lowerCAmelCase__	)	)

																						shutil.rmtree(lowerCAmelCase__	)




@require_torch
@require_sentencepiece
@require_tokenizers
class        _lowercase      (			unittest.TestCase       ):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE__				:			Optional[int]						=		'''facebook/mbart-large-en-ro'''
				SCREAMING_SNAKE_CASE__				:			Optional[int]						=		[
				    ''' UN Chief Says There Is No Military Solution in Syria''',
				    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
				]
				SCREAMING_SNAKE_CASE__				:			Any						=		[
				    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
				    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
				    ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
				    ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
				]
				SCREAMING_SNAKE_CASE__				:			Tuple						=		[8_274, 127_873, 25_916, 7, 8_622, 2_071, 438, 67_485, 53, 187_895, 23, 51_712, 2, EN_CODE]


				@classmethod
				def 		__magic_name__(       cls			:str	)		->			int:
										__SCREAMING_SNAKE_CASE						:     MBartTokenizer					=		MBartTokenizer.from_pretrained(
										    cls.checkpoint_name			,			src_lang='''en_XX'''			,			tgt_lang='''ro_RO'''	)
										__SCREAMING_SNAKE_CASE						:     List[Any]					=		1
										return cls


				def 		__magic_name__(       self			:Optional[Any]	)		->			Dict:
										self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ar_AR''']			,			250_001	)
										self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''en_EN''']			,			250_004	)
										self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ro_RO''']			,			250_020	)


				def 		__magic_name__(       self			:Any	)		->			Union[str, Any]:
										__SCREAMING_SNAKE_CASE						:     List[Any]					=		self.tokenizer.batch_encode_plus(self.src_text	).input_ids[0]
										self.assertListEqual(self.expected_src_tokens			,			lowerCAmelCase__	)


				def 		__magic_name__(       self			:List[Any]	)		->			Tuple:
										self.assertIn(lowerCAmelCase__			,			self.tokenizer.all_special_ids	)
										__SCREAMING_SNAKE_CASE						:     int					=		[RO_CODE, 884, 9_019, 96, 9, 916, 86_792, 36, 18_743, 15_596, 5, 2]
										__SCREAMING_SNAKE_CASE						:     str					=		self.tokenizer.decode(lowerCAmelCase__			,			skip_special_tokens=lowerCAmelCase__	)
										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		self.tokenizer.decode(generated_ids[1:]			,			skip_special_tokens=lowerCAmelCase__	)
										self.assertEqual(lowerCAmelCase__			,			lowerCAmelCase__	)
										self.assertNotIn(self.tokenizer.eos_token			,			lowerCAmelCase__	)


				def 		__magic_name__(       self			:List[str]	)		->			List[Any]:
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		['''this is gunna be a long sentence ''' * 20]
										assert isinstance(src_text[0]			,			lowerCAmelCase__	)
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		10
										__SCREAMING_SNAKE_CASE						:     List[Any]					=		self.tokenizer(lowerCAmelCase__			,			max_length=lowerCAmelCase__			,			truncation=lowerCAmelCase__	).input_ids[0]
										self.assertEqual(ids[-2]			,			2	)
										self.assertEqual(ids[-1]			,			lowerCAmelCase__	)
										self.assertEqual(len(lowerCAmelCase__	)			,			lowerCAmelCase__	)


				def 		__magic_name__(       self			:Any	)		->			List[str]:
										self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']	)			,			[250_026, 250_001]	)


				def 		__magic_name__(       self			:List[str]	)		->			List[Any]:
										__SCREAMING_SNAKE_CASE						:     Any					=		tempfile.mkdtemp()
										__SCREAMING_SNAKE_CASE						:     Dict					=		self.tokenizer.fairseq_tokens_to_ids
										self.tokenizer.save_pretrained(lowerCAmelCase__	)
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		MBartTokenizer.from_pretrained(lowerCAmelCase__	)
										self.assertDictEqual(new_tok.fairseq_tokens_to_ids			,			lowerCAmelCase__	)


				@require_torch
				def 		__magic_name__(       self			:Dict	)		->			List[Any]:
										__SCREAMING_SNAKE_CASE						:     List[str]					=		self.tokenizer(self.src_text			,			text_target=self.tgt_text			,			padding=lowerCAmelCase__			,			return_tensors='''pt'''	)
										__SCREAMING_SNAKE_CASE						:     List[Any]					=		shift_tokens_right(batch['''labels''']			,			self.tokenizer.pad_token_id	)

										# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
										assert batch.input_ids[1][-2:].tolist() == [2, EN_CODE]
										assert batch.decoder_input_ids[1][0].tolist() == RO_CODE
										assert batch.decoder_input_ids[1][-1] == 2
										assert batch.labels[1][-2:].tolist() == [2, RO_CODE]


				@require_torch
				def 		__magic_name__(       self			:Union[str, Any]	)		->			Tuple:
										__SCREAMING_SNAKE_CASE						:     int					=		self.tokenizer(
										    self.src_text			,			text_target=self.tgt_text			,			padding=lowerCAmelCase__			,			truncation=lowerCAmelCase__			,			max_length=len(self.expected_src_tokens	)			,			return_tensors='''pt'''			,			)

										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		shift_tokens_right(batch['''labels''']			,			self.tokenizer.pad_token_id	)

										self.assertIsInstance(lowerCAmelCase__			,			lowerCAmelCase__	)

										self.assertEqual((2, 14)			,			batch.input_ids.shape	)
										self.assertEqual((2, 14)			,			batch.attention_mask.shape	)
										__SCREAMING_SNAKE_CASE						:     str					=		batch.input_ids.tolist()[0]
										self.assertListEqual(self.expected_src_tokens			,			lowerCAmelCase__	)
										self.assertEqual(2			,			batch.decoder_input_ids[0, -1]	)  # EOS
										# Test that special tokens are reset
										self.assertEqual(self.tokenizer.prefix_tokens			,			[]	)
										self.assertEqual(self.tokenizer.suffix_tokens			,			[self.tokenizer.eos_token_id, EN_CODE]	)


				def 		__magic_name__(       self			:Optional[Any]	)		->			Optional[Any]:
										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		self.tokenizer(self.src_text			,			padding=lowerCAmelCase__			,			truncation=lowerCAmelCase__			,			max_length=3			,			return_tensors='''pt'''	)
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		self.tokenizer(
										    text_target=self.tgt_text			,			padding=lowerCAmelCase__			,			truncation=lowerCAmelCase__			,			max_length=10			,			return_tensors='''pt'''	)
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		targets['''input_ids''']
										__SCREAMING_SNAKE_CASE						:     Any					=		shift_tokens_right(lowerCAmelCase__			,			self.tokenizer.pad_token_id	)

										self.assertEqual(batch.input_ids.shape[1]			,			3	)
										self.assertEqual(batch.decoder_input_ids.shape[1]			,			10	)



				@require_torch
				def 		__magic_name__(       self			:Optional[int]	)		->			Dict:
										__SCREAMING_SNAKE_CASE						:     Dict					=		self.tokenizer._build_translation_inputs(
										    '''A test'''			,			return_tensors='''pt'''			,			src_lang='''en_XX'''			,			tgt_lang='''ar_AR'''	)

										self.assertEqual(
										    nested_simplify(lowerCAmelCase__	)			,			{
										        # A, test, EOS, en_XX
										        '''input_ids''': [[62, 3_034, 2, 250_004]],
										        '''attention_mask''': [[1, 1, 1, 1]],
										        # ar_AR
										        '''forced_bos_token_id''': 250_001,
										    }			,			)




from importlib import import_module

from .logging import get_logger


__lowerCAmelCase	: str  				=get_logger(__name__)
class        _lowercase      :






				'''simple docstring'''



				def __init__(       self			:List[Any]			,			lowerCAmelCase__			:List[Any]			,			lowerCAmelCase__			:str=None	)		->			int:
										__SCREAMING_SNAKE_CASE						:     List[str]					=		attrs or []
										if module is not None:
																for key in module.__dict__:
																						if key in attrs or not key.startswith('''__'''	):
																												setattr(self			,			lowerCAmelCase__			,			getattr(lowerCAmelCase__			,			lowerCAmelCase__	)	)
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		module._original_module if isinstance(lowerCAmelCase__			,			_PatchedModuleObj	) else module




class        _lowercase      :






				'''simple docstring'''



				SCREAMING_SNAKE_CASE__				:			Optional[int]						=		[]


				def __init__(       self			:Tuple			,			lowerCAmelCase__			:Optional[Any]			,			lowerCAmelCase__			:str			,			lowerCAmelCase__			:Any			,			lowerCAmelCase__			:Dict=None	)		->			List[Any]:
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		obj
										__SCREAMING_SNAKE_CASE						:     str					=		target
										__SCREAMING_SNAKE_CASE						:     Dict					=		new
										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		target.split('''.'''	)[0]
										__SCREAMING_SNAKE_CASE						:     List[str]					=		{}
										__SCREAMING_SNAKE_CASE						:     Tuple					=		attrs or []


				def __enter__(       self			:int	)		->			Dict:
										*__SCREAMING_SNAKE_CASE			,			__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		self.target.split('''.'''	)

										# Patch modules:
										# it's used to patch attributes of submodules like "os.path.join";
										# in this case we need to patch "os" and "os.path"

										for i in range(len(lowerCAmelCase__	)	):
																try:
																						__SCREAMING_SNAKE_CASE						:     Any					=		import_module('''.'''.join(submodules[: i + 1]	)	)
																except ModuleNotFoundError:
																						continue
																# We iterate over all the globals in self.obj in case we find "os" or "os.path"
																for attr in self.obj.__dir__():
																						__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		getattr(self.obj			,			lowerCAmelCase__	)
																						# We don't check for the name of the global, but rather if its value *is* "os" or "os.path".
																						# This allows to patch renamed modules like "from os import path as ospath".
																						if obj_attr is submodule or (
																						    (isinstance(lowerCAmelCase__			,			_PatchedModuleObj	) and obj_attr._original_module is submodule)
																						):
																												__SCREAMING_SNAKE_CASE						:     int					=		obj_attr
																												# patch at top level
																												setattr(self.obj			,			lowerCAmelCase__			,			_PatchedModuleObj(lowerCAmelCase__			,			attrs=self.attrs	)	)
																												__SCREAMING_SNAKE_CASE						:     List[str]					=		getattr(self.obj			,			lowerCAmelCase__	)
																												# construct lower levels patches
																												for key in submodules[i + 1 :]:
																																		setattr(lowerCAmelCase__			,			lowerCAmelCase__			,			_PatchedModuleObj(getattr(lowerCAmelCase__			,			lowerCAmelCase__			,			lowerCAmelCase__	)			,			attrs=self.attrs	)	)
																																		__SCREAMING_SNAKE_CASE						:     Tuple					=		getattr(lowerCAmelCase__			,			lowerCAmelCase__	)
																												# finally set the target attribute
																												setattr(lowerCAmelCase__			,			lowerCAmelCase__			,			self.new	)

        # Patch attribute itself:
        # it's used for builtins like "open",
        # and also to patch "os.path.join" we may also need to patch "join"
        # itself if it was imported as "from os.path import join".

										if submodules:  # if it's an attribute of a submodule like "os.path.join"
																try:
																						__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		getattr(import_module('''.'''.join(lowerCAmelCase__	)	)			,			lowerCAmelCase__	)
																except (AttributeError, ModuleNotFoundError):
																						return
																# We iterate over all the globals in self.obj in case we find "os.path.join"
																for attr in self.obj.__dir__():
																						# We don't check for the name of the global, but rather if its value *is* "os.path.join".
																						# This allows to patch renamed attributes like "from os.path import join as pjoin".
																						if getattr(self.obj			,			lowerCAmelCase__	) is attr_value:
																												__SCREAMING_SNAKE_CASE						:     Any					=		getattr(self.obj			,			lowerCAmelCase__	)
																												setattr(self.obj			,			lowerCAmelCase__			,			self.new	)
										elif target_attr in globals()["__builtins__"]:  # if it'a s builtin like "open"
																__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		globals()['''__builtins__'''][target_attr]
																setattr(self.obj			,			lowerCAmelCase__			,			self.new	)
										else:
																raise RuntimeError(f'''Tried to patch attribute {target_attr} instead of a submodule.'''	)


				def __exit__(       self			:str			,			*lowerCAmelCase__			:Union[str, Any]	)		->			Optional[int]:
										for attr in list(self.original	):
																setattr(self.obj			,			lowerCAmelCase__			,			self.original.pop(lowerCAmelCase__	)	)


				def 		__magic_name__(       self			:List[Any]	)		->			List[Any]:
										self.__enter__()
										self._active_patches.append(self	)



				def 		__magic_name__(       self			:Optional[int]	)		->			int:
										try:
																self._active_patches.remove(self	)
										except ValueError:
																# If the patch hasn't been started this will fail
																return None

										return self.__exit__()
