
import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


__A				=get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
			from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

__A				=2_5_6_0_4_7
__A				=2_5_6_1_4_5




@require_sentencepiece
@require_tokenizers
class     _SCREAMING_SNAKE_CASE     (      snake_case_			,       unittest.TestCase						):
	lowerCAmelCase__         = NllbTokenizer
	lowerCAmelCase__         = NllbTokenizerFast
	lowerCAmelCase__         = True
	lowerCAmelCase__         = True
	lowerCAmelCase__         = {}



	def     SCREAMING_SNAKE_CASE_(    self )				->       Dict:
					super().setUp()

					# We have a SentencePiece fixture for testing
					lowerCamelCase_						=			NllbTokenizer(lowercase						,  keep_accents=lowercase )
					tokenizer.save_pretrained(self.tmpdirname )



	def     SCREAMING_SNAKE_CASE_(    self )				->       Optional[int]:
					lowerCamelCase_						=			NllbTokenizer(lowercase						,  keep_accents=lowercase )

					lowerCamelCase_						=			tokenizer.tokenize("This is a test" )
					self.assertListEqual(lowercase						,  ["▁This", "▁is", "▁a", "▁t", "est"] )

					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(lowercase )						,  [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]						,  )

					lowerCamelCase_						=			tokenizer.tokenize("I was born in 92000, and this is falsé." )
					self.assertListEqual(
					    lowercase						,  [
					        SPIECE_UNDERLINE + "I",
					        SPIECE_UNDERLINE + "was",
					        SPIECE_UNDERLINE + "b",
					        "or",
					        "n",
					        SPIECE_UNDERLINE + "in",
					        SPIECE_UNDERLINE + "",
					        "9",
					        "2",
					        "0",
					        "0",
					        "0",
					        ",",
					        SPIECE_UNDERLINE + "and",
					        SPIECE_UNDERLINE + "this",
					        SPIECE_UNDERLINE + "is",
					        SPIECE_UNDERLINE + "f",
					        "al",
					        "s",
					        "é",
					        ".",
					    ]						,  )
					lowerCamelCase_						=			tokenizer.convert_tokens_to_ids(lowercase )
					self.assertListEqual(
					    lowercase						,  [
					        value + tokenizer.fairseq_offset
					        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
					    ]						,  )

					lowerCamelCase_						=			tokenizer.convert_ids_to_tokens(lowercase )
					self.assertListEqual(
					    lowercase						,  [
					        SPIECE_UNDERLINE + "I",
					        SPIECE_UNDERLINE + "was",
					        SPIECE_UNDERLINE + "b",
					        "or",
					        "n",
					        SPIECE_UNDERLINE + "in",
					        SPIECE_UNDERLINE + "",
					        "<unk>",
					        "2",
					        "0",
					        "0",
					        "0",
					        ",",
					        SPIECE_UNDERLINE + "and",
					        SPIECE_UNDERLINE + "this",
					        SPIECE_UNDERLINE + "is",
					        SPIECE_UNDERLINE + "f",
					        "al",
					        "s",
					        "<unk>",
					        ".",
					    ]						,  )



	def     SCREAMING_SNAKE_CASE_(    self )				->       List[str]:
					lowerCamelCase_						=			(self.rust_tokenizer_class, "hf-internal-testing/tiny-random-nllb", {})
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})' ):
													lowerCamelCase_						=			self.rust_tokenizer_class.from_pretrained(lowercase						,  **lowercase )
													lowerCamelCase_						=			self.tokenizer_class.from_pretrained(lowercase						,  **lowercase )

													lowerCamelCase_						=			tempfile.mkdtemp()

													lowerCamelCase_						=			tokenizer_r.save_pretrained(lowercase )
													lowerCamelCase_						=			tokenizer_p.save_pretrained(lowercase )

													# Checks it save with the same files + the tokenizer.json file for the fast one
													self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files ) )
													lowerCamelCase_						=			tuple(f for f in tokenizer_r_files if "tokenizer.json" not in f )
													self.assertSequenceEqual(lowercase						,  lowercase )

													# Checks everything loads correctly in the same way
													lowerCamelCase_						=			tokenizer_r.from_pretrained(lowercase )
													lowerCamelCase_						=			tokenizer_p.from_pretrained(lowercase )

													# Check special tokens are set accordingly on Rust and Python
													for key in tokenizer_pp.special_tokens_map:
																	self.assertTrue(hasattr(lowercase						,  lowercase ) )

													shutil.rmtree(lowercase )

													# Save tokenizer rust, legacy_format=True
													lowerCamelCase_						=			tempfile.mkdtemp()

													lowerCamelCase_						=			tokenizer_r.save_pretrained(lowercase						,  legacy_format=lowercase )
													lowerCamelCase_						=			tokenizer_p.save_pretrained(lowercase )

													# Checks it save with the same files
													self.assertSequenceEqual(lowercase						,  lowercase )

													# Checks everything loads correctly in the same way
													lowerCamelCase_						=			tokenizer_r.from_pretrained(lowercase )
													lowerCamelCase_						=			tokenizer_p.from_pretrained(lowercase )

													# Check special tokens are set accordingly on Rust and Python
													for key in tokenizer_pp.special_tokens_map:
																	self.assertTrue(hasattr(lowercase						,  lowercase ) )

													shutil.rmtree(lowercase )

													# Save tokenizer rust, legacy_format=False
													lowerCamelCase_						=			tempfile.mkdtemp()

													lowerCamelCase_						=			tokenizer_r.save_pretrained(lowercase						,  legacy_format=lowercase )
													lowerCamelCase_						=			tokenizer_p.save_pretrained(lowercase )

													# Checks it saved the tokenizer.json file
													self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files ) )

													# Checks everything loads correctly in the same way
													lowerCamelCase_						=			tokenizer_r.from_pretrained(lowercase )
													lowerCamelCase_						=			tokenizer_p.from_pretrained(lowercase )

													# Check special tokens are set accordingly on Rust and Python
													for key in tokenizer_pp.special_tokens_map:
																	self.assertTrue(hasattr(lowercase						,  lowercase ) )

													shutil.rmtree(lowercase )



	@require_torch
	def     SCREAMING_SNAKE_CASE_(    self )				->       str:
					if not self.test_seqaseq:
									return

					lowerCamelCase_						=			self.get_tokenizers()
					for tokenizer in tokenizers:
									with self.subTest(f'{tokenizer.__class__.__name__}' ):
													# Longer text that will definitely require truncation.
													lowerCamelCase_						=			[
													    " UN Chief Says There Is No Military Solution in Syria",
													    " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"
													    " Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"
													    " will only worsen the violence and misery for millions of people.",
													]
													lowerCamelCase_						=			[
													    "Şeful ONU declară că nu există o soluţie militară în Siria",
													    "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"
													    " Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"
													    " că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
													]
													try:
																	lowerCamelCase_						=			tokenizer.prepare_seqaseq_batch(
																	    src_texts=lowercase						,  tgt_texts=lowercase						,  max_length=3						,  max_target_length=10						,  return_tensors="pt"						,  src_lang="eng_Latn"						,  tgt_lang="ron_Latn"						,  )
													except NotImplementedError:
																	return
													self.assertEqual(batch.input_ids.shape[1]						,  3 )
													self.assertEqual(batch.labels.shape[1]						,  10 )
													# max_target_length will default to max_length if not specified
													lowerCamelCase_						=			tokenizer.prepare_seqaseq_batch(
													    lowercase						,  tgt_texts=lowercase						,  max_length=3						,  return_tensors="pt" )
													self.assertEqual(batch.input_ids.shape[1]						,  3 )
													self.assertEqual(batch.labels.shape[1]						,  3 )

													lowerCamelCase_						=			tokenizer.prepare_seqaseq_batch(
													    src_texts=lowercase						,  max_length=3						,  max_target_length=10						,  return_tensors="pt" )
													self.assertEqual(batch_encoder_only.input_ids.shape[1]						,  3 )
													self.assertEqual(batch_encoder_only.attention_mask.shape[1]						,  3 )
													self.assertNotIn("decoder_input_ids"						,  lowercase )



	@unittest.skip("Unfortunately way too slow to build a BPE with SentencePiece." )
	def     SCREAMING_SNAKE_CASE_(    self )				->       Any:
					pass



	def     SCREAMING_SNAKE_CASE_(    self )				->       List[Any]:
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})' ):
													lowerCamelCase_						=			[AddedToken("<special>"						,  lstrip=lowercase )]

													lowerCamelCase_						=			self.rust_tokenizer_class.from_pretrained(
													    lowercase						,  additional_special_tokens=lowercase						,  **lowercase )
													lowerCamelCase_						=			tokenizer_r.encode("Hey this is a <special> token" )

													lowerCamelCase_						=			tokenizer_r.encode("<special>"						,  add_special_tokens=lowercase )[0]

													self.assertTrue(special_token_id in r_output )

													if self.test_slow_tokenizer:
																	lowerCamelCase_						=			self.rust_tokenizer_class.from_pretrained(
																	    lowercase						,  additional_special_tokens=lowercase						,  **lowercase						,  )
																	lowerCamelCase_						=			self.tokenizer_class.from_pretrained(
																	    lowercase						,  additional_special_tokens=lowercase						,  **lowercase )

																	lowerCamelCase_						=			tokenizer_p.encode("Hey this is a <special> token" )

																	lowerCamelCase_						=			tokenizer_cr.encode("Hey this is a <special> token" )

																	self.assertEqual(lowercase						,  lowercase )
																	self.assertEqual(lowercase						,  lowercase )
																	self.assertTrue(special_token_id in p_output )
																	self.assertTrue(special_token_id in cr_output )




@require_torch
@require_sentencepiece
@require_tokenizers
class     _SCREAMING_SNAKE_CASE     (      unittest.TestCase						):
	lowerCAmelCase__         = 'facebook/nllb-200-distilled-600M'
	lowerCAmelCase__         = [
	    ' UN Chief Says There Is No Military Solution in Syria',
	    ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.',
	]
	lowerCAmelCase__         = [
	    'Şeful ONU declară că nu există o soluţie militară în Siria',
	    'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'
	    ' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'
	    ' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
	]
	lowerCAmelCase__         = [
	    25_60_47,
	    1_62_97,
	    13_44_08,
	    81_65,
	    24_80_66,
	    1_47_34,
	    9_50,
	    11_35,
	    10_57_21,
	    35_73,
	    83,
	    2_73_52,
	    1_08,
	    4_94_86,
	    2,
	]



	@classmethod
	def     SCREAMING_SNAKE_CASE_(    cls )				->       int:
					lowerCamelCase_						=			NllbTokenizer.from_pretrained(
					    cls.checkpoint_name						,  src_lang="eng_Latn"						,  tgt_lang="ron_Latn" )
					lowerCamelCase_						=			1
					return cls



	def     SCREAMING_SNAKE_CASE_(    self )				->       Optional[int]:
					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ace_Arab"]						,  256001 )
					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ace_Latn"]						,  256002 )
					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["fra_Latn"]						,  256057 )



	def     SCREAMING_SNAKE_CASE_(    self )				->       Any:
					lowerCamelCase_						=			self.tokenizer.batch_encode_plus(self.src_text ).input_ids[0]
					self.assertListEqual(self.expected_src_tokens						,  lowercase )



	def     SCREAMING_SNAKE_CASE_(    self )				->       Optional[Any]:
					self.assertIn(lowercase						,  self.tokenizer.all_special_ids )
					# fmt: off
					lowerCamelCase_						=			[RO_CODE, 4254, 98068, 112923, 39072, 3909, 713, 102767, 26, 17314, 35642, 14683, 33118, 2022, 66987, 2, 256047]
					# fmt: on

					lowerCamelCase_						=			self.tokenizer.decode(lowercase						,  skip_special_tokens=lowercase )
					lowerCamelCase_						=			self.tokenizer.decode(generated_ids[1:]						,  skip_special_tokens=lowercase )
					self.assertEqual(lowercase						,  lowercase )
					self.assertNotIn(self.tokenizer.eos_token						,  lowercase )



	def     SCREAMING_SNAKE_CASE_(    self )				->       Any:
					lowerCamelCase_						=			["this is gunna be a long sentence " * 20]
					assert isinstance(src_text[0]						,  lowercase )
					lowerCamelCase_						=			10
					lowerCamelCase_						=			self.tokenizer(lowercase						,  max_length=lowercase						,  truncation=lowercase ).input_ids[0]
					self.assertEqual(ids[-1]						,  2 )
					self.assertEqual(ids[0]						,  lowercase )
					self.assertEqual(len(lowercase )						,  lowercase )



	def     SCREAMING_SNAKE_CASE_(    self )				->       Any:
					self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "ar_AR"] )						,  [256203, 3] )



	def     SCREAMING_SNAKE_CASE_(    self )				->       Optional[int]:
					lowerCamelCase_						=			tempfile.mkdtemp()
					lowerCamelCase_						=			self.tokenizer.fairseq_tokens_to_ids
					self.tokenizer.save_pretrained(lowercase )
					lowerCamelCase_						=			NllbTokenizer.from_pretrained(lowercase )
					self.assertDictEqual(new_tok.fairseq_tokens_to_ids						,  lowercase )



	@require_torch
	def     SCREAMING_SNAKE_CASE_(    self )				->       int:
					lowerCamelCase_						=			self.tokenizer(
					    self.src_text						,  text_target=self.tgt_text						,  padding=lowercase						,  truncation=lowercase						,  max_length=len(self.expected_src_tokens )						,  return_tensors="pt"						,  )
					lowerCamelCase_						=			shift_tokens_right(
					    batch["labels"]						,  self.tokenizer.pad_token_id						,  self.tokenizer.lang_code_to_id["ron_Latn"] )

					self.assertIsInstance(lowercase						,  lowercase )

					self.assertEqual((2, 15)						,  batch.input_ids.shape )
					self.assertEqual((2, 15)						,  batch.attention_mask.shape )
					lowerCamelCase_						=			batch.input_ids.tolist()[0]
					self.assertListEqual(self.expected_src_tokens						,  lowercase )
					self.assertEqual(lowercase						,  batch.decoder_input_ids[0, 0] )  # EOS
					# Test that special tokens are reset
					self.assertEqual(self.tokenizer.prefix_tokens						,  [EN_CODE] )
					self.assertEqual(self.tokenizer.suffix_tokens						,  [self.tokenizer.eos_token_id] )



	def     SCREAMING_SNAKE_CASE_(    self )				->       int:
					lowerCamelCase_						=			self.tokenizer(self.src_text						,  padding=lowercase						,  truncation=lowercase						,  max_length=3						,  return_tensors="pt" )
					lowerCamelCase_						=			self.tokenizer(
					    text_target=self.tgt_text						,  padding=lowercase						,  truncation=lowercase						,  max_length=10						,  return_tensors="pt" )
					lowerCamelCase_						=			targets["input_ids"]
					lowerCamelCase_						=			shift_tokens_right(
					    lowercase						,  self.tokenizer.pad_token_id						,  decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]						,  )

					self.assertEqual(batch.input_ids.shape[1]						,  3 )
					self.assertEqual(batch.decoder_input_ids.shape[1]						,  10 )



	@require_torch
	def     SCREAMING_SNAKE_CASE_(    self )				->       List[Any]:
					lowerCamelCase_						=			self.tokenizer._build_translation_inputs(
					    "A test"						,  return_tensors="pt"						,  src_lang="eng_Latn"						,  tgt_lang="fra_Latn" )

					self.assertEqual(
					    nested_simplify(lowercase )						,  {
					        # A, test, EOS, en_XX
					        "input_ids": [[256047, 70, 7356, 2]],
					        "attention_mask": [[1, 1, 1, 1]],
					        # ar_AR
					        "forced_bos_token_id": 256057,
					    }						,  )



	@require_torch
	def     SCREAMING_SNAKE_CASE_(    self )				->       Tuple:
					lowerCamelCase_						=			True
					lowerCamelCase_						=			self.tokenizer(
					    "UN Chief says there is no military solution in Syria"						,  src_lang="eng_Latn"						,  tgt_lang="fra_Latn" )
					self.assertEqual(
					    inputs.input_ids						,  [16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2, 256047] )

					lowerCamelCase_						=			False
					lowerCamelCase_						=			self.tokenizer(
					    "UN Chief says there is no military solution in Syria"						,  src_lang="eng_Latn"						,  tgt_lang="fra_Latn" )
					self.assertEqual(
					    inputs.input_ids						,  [256047, 16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2] )




__A				={str(digit): digit**5 for digit in range(1_0)}





def 			lowerCamelCase_ (				lowerCamelCase__    ):
				return sum(DIGITS_FIFTH_POWER[digit] for digit in str(lowerCamelCase__    )    )





def 			lowerCamelCase_ (				):
				return sum(
				    number
				    for number in range(1_0_0_0	, 1_0_0_0_0_0_0    )
				    if number == digits_fifth_powers_sum(lowerCamelCase__    )    )


if __name__ == "__main__":
			print(solution())


