


import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase           =		get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
	from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

lowerCAmelCase           =		2_5_6_0_4_7
lowerCAmelCase           =		2_5_6_1_4_5





@require_sentencepiece
@require_tokenizers
class       A    (   _a					,				unittest.TestCase ):
							UpperCamelCase_   :							Union[str, Any]       =NllbTokenizer
							UpperCamelCase_   :							Optional[int]       =NllbTokenizerFast
							UpperCamelCase_   :							str       =True
							UpperCamelCase_   :							Tuple       =True
							UpperCamelCase_   :							Dict       ={}


							def 			_A       (self					):
									super().setUp()

									# We have a SentencePiece fixture for testing
									__lowercase=			NllbTokenizer(UpperCamelCase__      ,				keep_accents=UpperCamelCase__					)
									tokenizer.save_pretrained(self.tmpdirname					)


							def 			_A       (self					):
									__lowercase=			NllbTokenizer(UpperCamelCase__      ,				keep_accents=UpperCamelCase__					)

									__lowercase=			tokenizer.tokenize('This is a test'					)
									self.assertListEqual(UpperCamelCase__      ,				['▁This', '▁is', '▁a', '▁t', 'est']					)

									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(UpperCamelCase__					)      ,				[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]      ,				)

									__lowercase=			tokenizer.tokenize('I was born in 92000, and this is falsé.'					)
									self.assertListEqual(
									    UpperCamelCase__      ,				[
									        SPIECE_UNDERLINE + 'I',
									        SPIECE_UNDERLINE + 'was',
									        SPIECE_UNDERLINE + 'b',
									        'or',
									        'n',
									        SPIECE_UNDERLINE + 'in',
									        SPIECE_UNDERLINE + '',
									        '9',
									        '2',
									        '0',
									        '0',
									        '0',
									        ',',
									        SPIECE_UNDERLINE + 'and',
									        SPIECE_UNDERLINE + 'this',
									        SPIECE_UNDERLINE + 'is',
									        SPIECE_UNDERLINE + 'f',
									        'al',
									        's',
									        'é',
									        '.',
									    ]      ,				)
									__lowercase=			tokenizer.convert_tokens_to_ids(UpperCamelCase__					)
									self.assertListEqual(
									    UpperCamelCase__      ,				[
									        value + tokenizer.fairseq_offset
									        for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
									    ]      ,				)

									__lowercase=			tokenizer.convert_ids_to_tokens(UpperCamelCase__					)
									self.assertListEqual(
									    UpperCamelCase__      ,				[
									        SPIECE_UNDERLINE + 'I',
									        SPIECE_UNDERLINE + 'was',
									        SPIECE_UNDERLINE + 'b',
									        'or',
									        'n',
									        SPIECE_UNDERLINE + 'in',
									        SPIECE_UNDERLINE + '',
									        '<unk>',
									        '2',
									        '0',
									        '0',
									        '0',
									        ',',
									        SPIECE_UNDERLINE + 'and',
									        SPIECE_UNDERLINE + 'this',
									        SPIECE_UNDERLINE + 'is',
									        SPIECE_UNDERLINE + 'f',
									        'al',
									        's',
									        '<unk>',
									        '.',
									    ]      ,				)


							def 			_A       (self					):
									__lowercase=			(self.rust_tokenizer_class, 'hf-internal-testing/tiny-random-nllb', {})
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'					):
													__lowercase=			self.rust_tokenizer_class.from_pretrained(UpperCamelCase__      ,				**UpperCamelCase__					)
													__lowercase=			self.tokenizer_class.from_pretrained(UpperCamelCase__      ,				**UpperCamelCase__					)

													__lowercase=			tempfile.mkdtemp()

													__lowercase=			tokenizer_r.save_pretrained(UpperCamelCase__					)
													__lowercase=			tokenizer_p.save_pretrained(UpperCamelCase__					)

													# Checks it save with the same files + the tokenizer.json file for the fast one
													self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files					)					)
													__lowercase=			tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f					)
													self.assertSequenceEqual(UpperCamelCase__      ,				UpperCamelCase__					)

													# Checks everything loads correctly in the same way
													__lowercase=			tokenizer_r.from_pretrained(UpperCamelCase__					)
													__lowercase=			tokenizer_p.from_pretrained(UpperCamelCase__					)

													# Check special tokens are set accordingly on Rust and Python
													for key in tokenizer_pp.special_tokens_map:
															self.assertTrue(hasattr(UpperCamelCase__      ,				UpperCamelCase__					)					)

													shutil.rmtree(UpperCamelCase__					)

													# Save tokenizer rust, legacy_format=True
													__lowercase=			tempfile.mkdtemp()

													__lowercase=			tokenizer_r.save_pretrained(UpperCamelCase__      ,				legacy_format=UpperCamelCase__					)
													__lowercase=			tokenizer_p.save_pretrained(UpperCamelCase__					)

													# Checks it save with the same files
													self.assertSequenceEqual(UpperCamelCase__      ,				UpperCamelCase__					)

													# Checks everything loads correctly in the same way
													__lowercase=			tokenizer_r.from_pretrained(UpperCamelCase__					)
													__lowercase=			tokenizer_p.from_pretrained(UpperCamelCase__					)

													# Check special tokens are set accordingly on Rust and Python
													for key in tokenizer_pp.special_tokens_map:
															self.assertTrue(hasattr(UpperCamelCase__      ,				UpperCamelCase__					)					)

													shutil.rmtree(UpperCamelCase__					)

													# Save tokenizer rust, legacy_format=False
													__lowercase=			tempfile.mkdtemp()

													__lowercase=			tokenizer_r.save_pretrained(UpperCamelCase__      ,				legacy_format=UpperCamelCase__					)
													__lowercase=			tokenizer_p.save_pretrained(UpperCamelCase__					)

													# Checks it saved the tokenizer.json file
													self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files					)					)

													# Checks everything loads correctly in the same way
													__lowercase=			tokenizer_r.from_pretrained(UpperCamelCase__					)
													__lowercase=			tokenizer_p.from_pretrained(UpperCamelCase__					)

													# Check special tokens are set accordingly on Rust and Python
													for key in tokenizer_pp.special_tokens_map:
															self.assertTrue(hasattr(UpperCamelCase__      ,				UpperCamelCase__					)					)

													shutil.rmtree(UpperCamelCase__					)


							@require_torch
							def 			_A       (self					):







									if not self.test_seqaseq:
											return

									__lowercase=			self.get_tokenizers()
									for tokenizer in tokenizers:
											with self.subTest(f'{tokenizer.__class__.__name__}'					):
													# Longer text that will definitely require truncation.
													__lowercase=			[
													    ' UN Chief Says There Is No Military Solution in Syria',
													    ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'
													    ' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'
													    ' will only worsen the violence and misery for millions of people.',
													]
													__lowercase=			[
													    'Şeful ONU declară că nu există o soluţie militară în Siria',
													    'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'
													    ' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'
													    ' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
													]
													try:
															__lowercase=			tokenizer.prepare_seqaseq_batch(
															    src_texts=UpperCamelCase__      ,				tgt_texts=UpperCamelCase__      ,				max_length=3      ,				max_target_length=1_0      ,				return_tensors='pt'      ,				src_lang='eng_Latn'      ,				tgt_lang='ron_Latn'      ,				)
													except NotImplementedError:
															return
													self.assertEqual(batch.input_ids.shape[1]      ,				3					)
													self.assertEqual(batch.labels.shape[1]      ,				1_0					)
													# max_target_length will default to max_length if not specified
													__lowercase=			tokenizer.prepare_seqaseq_batch(
													    UpperCamelCase__      ,				tgt_texts=UpperCamelCase__      ,				max_length=3      ,				return_tensors='pt'					)
													self.assertEqual(batch.input_ids.shape[1]      ,				3					)
													self.assertEqual(batch.labels.shape[1]      ,				3					)

													__lowercase=			tokenizer.prepare_seqaseq_batch(
													    src_texts=UpperCamelCase__      ,				max_length=3      ,				max_target_length=1_0      ,				return_tensors='pt'					)
													self.assertEqual(batch_encoder_only.input_ids.shape[1]      ,				3					)
													self.assertEqual(batch_encoder_only.attention_mask.shape[1]      ,				3					)
													self.assertNotIn('decoder_input_ids'      ,				UpperCamelCase__					)


							@unittest.skip('Unfortunately way too slow to build a BPE with SentencePiece.'					)
							def 			_A       (self					):
									pass







							def 			_A       (self					):







									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'					):
													__lowercase=			[AddedToken('<special>'      ,				lstrip=UpperCamelCase__					)]

													__lowercase=			self.rust_tokenizer_class.from_pretrained(
													    UpperCamelCase__      ,				additional_special_tokens=UpperCamelCase__      ,				**UpperCamelCase__					)
													__lowercase=			tokenizer_r.encode('Hey this is a <special> token'					)

													__lowercase=			tokenizer_r.encode('<special>'      ,				add_special_tokens=UpperCamelCase__					)[0]

													self.assertTrue(special_token_id in r_output					)

													if self.test_slow_tokenizer:
															__lowercase=			self.rust_tokenizer_class.from_pretrained(
															    UpperCamelCase__      ,				additional_special_tokens=UpperCamelCase__      ,				**UpperCamelCase__      ,				)
															__lowercase=			self.tokenizer_class.from_pretrained(
															    UpperCamelCase__      ,				additional_special_tokens=UpperCamelCase__      ,				**UpperCamelCase__					)

															__lowercase=			tokenizer_p.encode('Hey this is a <special> token'					)

															__lowercase=			tokenizer_cr.encode('Hey this is a <special> token'					)

															self.assertEqual(UpperCamelCase__      ,				UpperCamelCase__					)
															self.assertEqual(UpperCamelCase__      ,				UpperCamelCase__					)
															self.assertTrue(special_token_id in p_output					)
															self.assertTrue(special_token_id in cr_output					)





@require_torch
@require_sentencepiece
@require_tokenizers
class       A    (   unittest.TestCase ):
							UpperCamelCase_   :							List[str]       ='''facebook/nllb-200-distilled-600M'''
							UpperCamelCase_   :							str       =[
							    ''' UN Chief Says There Is No Military Solution in Syria''',
							    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
							]
							UpperCamelCase_   :							Dict       =[
							    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
							    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
							    ''' pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor'''
							    ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
							]
							UpperCamelCase_   :							Optional[Any]       =[
							    256_047,
							    16_297,
							    134_408,
							    8_165,
							    248_066,
							    14_734,
							    950,
							    1_135,
							    105_721,
							    3_573,
							    83,
							    27_352,
							    108,
							    49_486,
							    2,
							]


							@classmethod
							def 			_A       (cls					):
									__lowercase=			NllbTokenizer.from_pretrained(
									    cls.checkpoint_name      ,				src_lang='eng_Latn'      ,				tgt_lang='ron_Latn'					)
									__lowercase=			1
									return cls


							def 			_A       (self					):
									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Arab']      ,				2_5_6_0_0_1					)
									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Latn']      ,				2_5_6_0_0_2					)
									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['fra_Latn']      ,				2_5_6_0_5_7					)


							def 			_A       (self					):
									__lowercase=			self.tokenizer.batch_encode_plus(self.src_text					).input_ids[0]
									self.assertListEqual(self.expected_src_tokens      ,				UpperCamelCase__					)


							def 			_A       (self					):
									self.assertIn(UpperCamelCase__      ,				self.tokenizer.all_special_ids					)
									# fmt: off
									__lowercase=			[RO_CODE, 4_2_5_4, 9_8_0_6_8, 1_1_2_9_2_3, 3_9_0_7_2, 3_9_0_9, 7_1_3, 1_0_2_7_6_7, 2_6, 1_7_3_1_4, 3_5_6_4_2, 1_4_6_8_3, 3_3_1_1_8, 2_0_2_2, 6_6_9_8_7, 2, 2_5_6_0_4_7]
									# fmt: on

									__lowercase=			self.tokenizer.decode(UpperCamelCase__      ,				skip_special_tokens=UpperCamelCase__					)
									__lowercase=			self.tokenizer.decode(generated_ids[1:]      ,				skip_special_tokens=UpperCamelCase__					)
									self.assertEqual(UpperCamelCase__      ,				UpperCamelCase__					)
									self.assertNotIn(self.tokenizer.eos_token      ,				UpperCamelCase__					)


							def 			_A       (self					):
									__lowercase=			['this is gunna be a long sentence ' * 2_0]
									assert isinstance(src_text[0]      ,				UpperCamelCase__					)
									__lowercase=			1_0
									__lowercase=			self.tokenizer(UpperCamelCase__      ,				max_length=UpperCamelCase__      ,				truncation=UpperCamelCase__					).input_ids[0]
									self.assertEqual(ids[-1]      ,				2					)
									self.assertEqual(ids[0]      ,				UpperCamelCase__					)
									self.assertEqual(len(UpperCamelCase__					)      ,				UpperCamelCase__					)


							def 			_A       (self					):
									self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR']					)      ,				[2_5_6_2_0_3, 3]					)


							def 			_A       (self					):
									__lowercase=			tempfile.mkdtemp()
									__lowercase=			self.tokenizer.fairseq_tokens_to_ids
									self.tokenizer.save_pretrained(UpperCamelCase__					)
									__lowercase=			NllbTokenizer.from_pretrained(UpperCamelCase__					)
									self.assertDictEqual(new_tok.fairseq_tokens_to_ids      ,				UpperCamelCase__					)


							@require_torch
							def 			_A       (self					):
									__lowercase=			self.tokenizer(
									    self.src_text      ,				text_target=self.tgt_text      ,				padding=UpperCamelCase__      ,				truncation=UpperCamelCase__      ,				max_length=len(self.expected_src_tokens					)      ,				return_tensors='pt'      ,				)
									__lowercase=			shift_tokens_right(
									    batch['labels']      ,				self.tokenizer.pad_token_id      ,				self.tokenizer.lang_code_to_id['ron_Latn']					)

									self.assertIsInstance(UpperCamelCase__      ,				UpperCamelCase__					)

									self.assertEqual((2, 1_5)      ,				batch.input_ids.shape					)
									self.assertEqual((2, 1_5)      ,				batch.attention_mask.shape					)
									__lowercase=			batch.input_ids.tolist()[0]
									self.assertListEqual(self.expected_src_tokens      ,				UpperCamelCase__					)
									self.assertEqual(UpperCamelCase__      ,				batch.decoder_input_ids[0, 0]					)  # EOS
									# Test that special tokens are reset
									self.assertEqual(self.tokenizer.prefix_tokens      ,				[EN_CODE]					)
									self.assertEqual(self.tokenizer.suffix_tokens      ,				[self.tokenizer.eos_token_id]					)


							def 			_A       (self					):
									__lowercase=			self.tokenizer(self.src_text      ,				padding=UpperCamelCase__      ,				truncation=UpperCamelCase__      ,				max_length=3      ,				return_tensors='pt'					)
									__lowercase=			self.tokenizer(
									    text_target=self.tgt_text      ,				padding=UpperCamelCase__      ,				truncation=UpperCamelCase__      ,				max_length=1_0      ,				return_tensors='pt'					)
									__lowercase=			targets['input_ids']
									__lowercase=			shift_tokens_right(
									    UpperCamelCase__      ,				self.tokenizer.pad_token_id      ,				decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]      ,				)

									self.assertEqual(batch.input_ids.shape[1]      ,				3					)
									self.assertEqual(batch.decoder_input_ids.shape[1]      ,				1_0					)


							@require_torch
							def 			_A       (self					):
									__lowercase=			self.tokenizer._build_translation_inputs(
									    'A test'      ,				return_tensors='pt'      ,				src_lang='eng_Latn'      ,				tgt_lang='fra_Latn'					)

									self.assertEqual(
									    nested_simplify(UpperCamelCase__					)      ,				{
									        # A, test, EOS, en_XX
									        'input_ids': [[2_5_6_0_4_7, 7_0, 7_3_5_6, 2]],
									        'attention_mask': [[1, 1, 1, 1]],
									        # ar_AR
									        'forced_bos_token_id': 2_5_6_0_5_7,
									    }      ,				)







							@require_torch
							def 			_A       (self					):
									__lowercase=			True
									__lowercase=			self.tokenizer(
									    'UN Chief says there is no military solution in Syria'      ,				src_lang='eng_Latn'      ,				tgt_lang='fra_Latn'					)
									self.assertEqual(
									    inputs.input_ids      ,				[1_6_2_9_7, 1_3_4_4_0_8, 2_5_6_5_3, 6_3_7_0, 2_4_8, 2_5_4, 1_0_3_9_2_9, 9_4_9_9_5, 1_0_8, 4_9_4_8_6, 2, 2_5_6_0_4_7]					)

									__lowercase=			False
									__lowercase=			self.tokenizer(
									    'UN Chief says there is no military solution in Syria'      ,				src_lang='eng_Latn'      ,				tgt_lang='fra_Latn'					)
									self.assertEqual(
									    inputs.input_ids      ,				[2_5_6_0_4_7, 1_6_2_9_7, 1_3_4_4_0_8, 2_5_6_5_3, 6_3_7_0, 2_4_8, 2_5_4, 1_0_3_9_2_9, 9_4_9_9_5, 1_0_8, 4_9_4_8_6, 2]					)











'''simple docstring'''


_lowerCamelCase	:       int      	=	"0.21.0"

from .accelerator import Accelerator
from .big_modeling import (
    cpu_offload,
    cpu_offload_with_hook,
    disk_offload,
    dispatch_model,
    init_empty_weights,
    init_on_device,
    load_checkpoint_and_dispatch,
)
from .data_loader import skip_first_batches
from .launchers import debug_launcher, notebook_launcher
from .state import PartialState
from .utils import (
    DeepSpeedPlugin,
    DistributedDataParallelKwargs,
    DistributedType,
    FullyShardedDataParallelPlugin,
    GradScalerKwargs,
    InitProcessGroupKwargs,
    find_executable_batch_size,
    infer_auto_device_map,
    is_rich_available,
    load_checkpoint_in_model,
    synchronize_rng_states,
)


if is_rich_available():
   from .utils import rich

