


'''simple docstring'''
import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__    					=      get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
  from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

UpperCamelCase__    					=      2_5_6_0_4_7
UpperCamelCase__    					=      2_5_6_1_4_5




@require_sentencepiece
@require_tokenizers
class 			lowerCamelCase_   (	lowerCAmelCase__     ,       unittest.TestCase      ):
      lowerCAmelCase__		     =     NllbTokenizer
      lowerCAmelCase__		     =     NllbTokenizerFast
      lowerCAmelCase__		     =     True
      lowerCAmelCase__		     =     True
      lowerCAmelCase__		     =     {}






      def lowercase_							(					self				:    int ):


          '''simple docstring'''


          super().setUp()

          # We have a SentencePiece fixture for testing
          UpperCAmelCase__			:		Dict													=				NllbTokenizer(__a						,						keep_accents=__a )
          tokenizer.save_pretrained(self.tmpdirname )






      def lowercase_							(					self				:    List[str] ):


          '''simple docstring'''


          UpperCAmelCase__			:		List[str]													=				NllbTokenizer(__a						,						keep_accents=__a )

          UpperCAmelCase__			:		Tuple													=				tokenizer.tokenize('''This is a test''' )
          self.assertListEqual(__a						,						['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''] )

          self.assertListEqual(
              tokenizer.convert_tokens_to_ids(__a )						,						[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]						,						)

          UpperCAmelCase__			:		Tuple													=				tokenizer.tokenize('''I was born in 92000, and this is falsé.''' )
          self.assertListEqual(
              __a						,						[
                  SPIECE_UNDERLINE + '''I''',
                  SPIECE_UNDERLINE + '''was''',
                  SPIECE_UNDERLINE + '''b''',
                  '''or''',
                  '''n''',
                  SPIECE_UNDERLINE + '''in''',
                  SPIECE_UNDERLINE + '''''',
                  '''9''',
                  '''2''',
                  '''0''',
                  '''0''',
                  '''0''',
                  ''',''',
                  SPIECE_UNDERLINE + '''and''',
                  SPIECE_UNDERLINE + '''this''',
                  SPIECE_UNDERLINE + '''is''',
                  SPIECE_UNDERLINE + '''f''',
                  '''al''',
                  '''s''',
                  '''é''',
                  '''.''',
              ]						,						)
          UpperCAmelCase__			:		List[str]													=				tokenizer.convert_tokens_to_ids(__a )
          self.assertListEqual(
              __a						,						[
                  value + tokenizer.fairseq_offset
                  for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
              ]						,						)

          UpperCAmelCase__			:		Dict													=				tokenizer.convert_ids_to_tokens(__a )
          self.assertListEqual(
              __a						,						[
                  SPIECE_UNDERLINE + '''I''',
                  SPIECE_UNDERLINE + '''was''',
                  SPIECE_UNDERLINE + '''b''',
                  '''or''',
                  '''n''',
                  SPIECE_UNDERLINE + '''in''',
                  SPIECE_UNDERLINE + '''''',
                  '''<unk>''',
                  '''2''',
                  '''0''',
                  '''0''',
                  '''0''',
                  ''',''',
                  SPIECE_UNDERLINE + '''and''',
                  SPIECE_UNDERLINE + '''this''',
                  SPIECE_UNDERLINE + '''is''',
                  SPIECE_UNDERLINE + '''f''',
                  '''al''',
                  '''s''',
                  '''<unk>''',
                  '''.''',
              ]						,						)






      def lowercase_							(					self				:    Any ):


          '''simple docstring'''


          UpperCAmelCase__			:		List[Any]													=				(self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
                  UpperCAmelCase__			:		Optional[Any]													=				self.rust_tokenizer_class.from_pretrained(__a						,						**__a )
                  UpperCAmelCase__			:		Union[str, Any]													=				self.tokenizer_class.from_pretrained(__a						,						**__a )

                  UpperCAmelCase__			:		int													=				tempfile.mkdtemp()

                  UpperCAmelCase__			:		Optional[int]													=				tokenizer_r.save_pretrained(__a )
                  UpperCAmelCase__			:		Tuple													=				tokenizer_p.save_pretrained(__a )

                  # Checks it save with the same files + the tokenizer.json file for the fast one
                  self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files ) )
                  UpperCAmelCase__			:		Any													=				tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f )
                  self.assertSequenceEqual(__a						,						__a )

                  # Checks everything loads correctly in the same way
                  UpperCAmelCase__			:		Optional[int]													=				tokenizer_r.from_pretrained(__a )
                  UpperCAmelCase__			:		Dict													=				tokenizer_p.from_pretrained(__a )

                  # Check special tokens are set accordingly on Rust and Python
                  for key in tokenizer_pp.special_tokens_map:
                      self.assertTrue(hasattr(__a						,						__a ) )

                  shutil.rmtree(__a )

                  # Save tokenizer rust, legacy_format=True
                  UpperCAmelCase__			:		List[Any]													=				tempfile.mkdtemp()

                  UpperCAmelCase__			:		int													=				tokenizer_r.save_pretrained(__a						,						legacy_format=__a )
                  UpperCAmelCase__			:		Dict													=				tokenizer_p.save_pretrained(__a )

                  # Checks it save with the same files
                  self.assertSequenceEqual(__a						,						__a )

                  # Checks everything loads correctly in the same way
                  UpperCAmelCase__			:		Dict													=				tokenizer_r.from_pretrained(__a )
                  UpperCAmelCase__			:		Union[str, Any]													=				tokenizer_p.from_pretrained(__a )

                  # Check special tokens are set accordingly on Rust and Python
                  for key in tokenizer_pp.special_tokens_map:
                      self.assertTrue(hasattr(__a						,						__a ) )

                  shutil.rmtree(__a )

                  # Save tokenizer rust, legacy_format=False
                  UpperCAmelCase__			:		Tuple													=				tempfile.mkdtemp()

                  UpperCAmelCase__			:		Tuple													=				tokenizer_r.save_pretrained(__a						,						legacy_format=__a )
                  UpperCAmelCase__			:		Any													=				tokenizer_p.save_pretrained(__a )

                  # Checks it saved the tokenizer.json file
                  self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files ) )

                  # Checks everything loads correctly in the same way
                  UpperCAmelCase__			:		int													=				tokenizer_r.from_pretrained(__a )
                  UpperCAmelCase__			:		Any													=				tokenizer_p.from_pretrained(__a )

                  # Check special tokens are set accordingly on Rust and Python
                  for key in tokenizer_pp.special_tokens_map:
                      self.assertTrue(hasattr(__a						,						__a ) )

                  shutil.rmtree(__a )






      @require_torch
      def lowercase_							(					self				:    Optional[int] ):


          '''simple docstring'''


          if not self.test_seqaseq:
              return

          UpperCAmelCase__			:		Optional[int]													=				self.get_tokenizers()
          for tokenizer in tokenizers:
              with self.subTest(f"""{tokenizer.__class__.__name__}""" ):
                  # Longer text that will definitely require truncation.
                  UpperCAmelCase__			:		Optional[int]													=				[
                      ''' UN Chief Says There Is No Military Solution in Syria''',
                      ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
                      ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
                      ''' will only worsen the violence and misery for millions of people.''',
                  ]
                  UpperCAmelCase__			:		Dict													=				[
                      '''Şeful ONU declară că nu există o soluţie militară în Siria''',
                      '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
                      ''' Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi'''
                      ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
                  ]
                  try:
                      UpperCAmelCase__			:		Tuple													=				tokenizer.prepare_seqaseq_batch(
                          src_texts=__a						,						tgt_texts=__a						,						max_length=3						,						max_target_length=10						,						return_tensors='''pt'''						,						src_lang='''eng_Latn'''						,						tgt_lang='''ron_Latn'''						,						)
                  except NotImplementedError:
                      return
                  self.assertEqual(batch.input_ids.shape[1]						,						3 )
                  self.assertEqual(batch.labels.shape[1]						,						10 )
                  # max_target_length will default to max_length if not specified
                  UpperCAmelCase__			:		Dict													=				tokenizer.prepare_seqaseq_batch(
                      __a						,						tgt_texts=__a						,						max_length=3						,						return_tensors='''pt''' )
                  self.assertEqual(batch.input_ids.shape[1]						,						3 )
                  self.assertEqual(batch.labels.shape[1]						,						3 )

                  UpperCAmelCase__			:		int													=				tokenizer.prepare_seqaseq_batch(
                      src_texts=__a						,						max_length=3						,						max_target_length=10						,						return_tensors='''pt''' )
                  self.assertEqual(batch_encoder_only.input_ids.shape[1]						,						3 )
                  self.assertEqual(batch_encoder_only.attention_mask.shape[1]						,						3 )
                  self.assertNotIn('''decoder_input_ids'''						,						__a )






      @unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.''' )
      def lowercase_							(					self				:    Dict ):


          '''simple docstring'''


          pass






      def lowercase_							(					self				:    Optional[Any] ):


          '''simple docstring'''


          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
                  UpperCAmelCase__			:		Optional[int]													=				[AddedToken('''<special>'''						,						lstrip=__a )]

                  UpperCAmelCase__			:		Optional[int]													=				self.rust_tokenizer_class.from_pretrained(
                      __a						,						additional_special_tokens=__a						,						**__a )
                  UpperCAmelCase__			:		int													=				tokenizer_r.encode('''Hey this is a <special> token''' )

                  UpperCAmelCase__			:		List[Any]													=				tokenizer_r.encode('''<special>'''						,						add_special_tokens=__a )[0]

                  self.assertTrue(special_token_id in r_output )

                  if self.test_slow_tokenizer:
                      UpperCAmelCase__			:		Tuple													=				self.rust_tokenizer_class.from_pretrained(
                          __a						,						additional_special_tokens=__a						,						**__a						,						)
                      UpperCAmelCase__			:		Union[str, Any]													=				self.tokenizer_class.from_pretrained(
                          __a						,						additional_special_tokens=__a						,						**__a )

                      UpperCAmelCase__			:		Optional[Any]													=				tokenizer_p.encode('''Hey this is a <special> token''' )

                      UpperCAmelCase__			:		int													=				tokenizer_cr.encode('''Hey this is a <special> token''' )

                      self.assertEqual(__a						,						__a )
                      self.assertEqual(__a						,						__a )
                      self.assertTrue(special_token_id in p_output )
                      self.assertTrue(special_token_id in cr_output )




@require_torch
@require_sentencepiece
@require_tokenizers
class 			lowerCamelCase_   (	unittest.TestCase      ):
      lowerCAmelCase__		     =     """facebook/nllb-200-distilled-600M"""
      lowerCAmelCase__		     =     [
          """ UN Chief Says There Is No Military Solution in Syria""",
          """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
      ]
      lowerCAmelCase__		     =     [
          """Şeful ONU declară că nu există o soluţie militară în Siria""",
          """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"""
          """ pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"""
          """ face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
      ]
      lowerCAmelCase__		     =     [
          2_5_6_0_4_7,
          1_6_2_9_7,
          1_3_4_4_0_8,
          8_1_6_5,
          2_4_8_0_6_6,
          1_4_7_3_4,
          9_5_0,
          1_1_3_5,
          1_0_5_7_2_1,
          3_5_7_3,
          8_3,
          2_7_3_5_2,
          1_0_8,
          4_9_4_8_6,
          2,
      ]






      @classmethod
      def lowercase_							(					cls				:    str ):


          '''simple docstring'''


          UpperCAmelCase__			:		Dict													=				NllbTokenizer.from_pretrained(
              cls.checkpoint_name						,						src_lang='''eng_Latn'''						,						tgt_lang='''ron_Latn''' )
          UpperCAmelCase__			:		Dict													=				1
          return cls






      def lowercase_							(					self				:    Optional[int] ):


          '''simple docstring'''


          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab''']						,						256_001 )
          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn''']						,						256_002 )
          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn''']						,						256_057 )






      def lowercase_							(					self				:    Optional[int] ):


          '''simple docstring'''


          UpperCAmelCase__			:		Optional[int]													=				self.tokenizer.batch_encode_plus(self.src_text ).input_ids[0]
          self.assertListEqual(self.expected_src_tokens						,						__a )






      def lowercase_							(					self				:    Dict ):


          '''simple docstring'''


          self.assertIn(__a						,						self.tokenizer.all_special_ids )
          # fmt: off
          UpperCAmelCase__			:		Optional[int]													=				[RO_CODE, 4_254, 98_068, 112_923, 39_072, 3_909, 713, 102_767, 26, 17_314, 35_642, 14_683, 33_118, 2_022, 66_987, 2, 256_047]
          # fmt: on

          UpperCAmelCase__			:		Union[str, Any]													=				self.tokenizer.decode(__a						,						skip_special_tokens=__a )
          UpperCAmelCase__			:		Optional[int]													=				self.tokenizer.decode(generated_ids[1:]						,						skip_special_tokens=__a )
          self.assertEqual(__a						,						__a )
          self.assertNotIn(self.tokenizer.eos_token						,						__a )






      def lowercase_							(					self				:    List[str] ):


          '''simple docstring'''


          UpperCAmelCase__			:		str													=				['''this is gunna be a long sentence ''' * 20]
          assert isinstance(src_text[0]						,						__a )
          UpperCAmelCase__			:		List[str]													=				10
          UpperCAmelCase__			:		List[Any]													=				self.tokenizer(__a						,						max_length=__a						,						truncation=__a ).input_ids[0]
          self.assertEqual(ids[-1]						,						2 )
          self.assertEqual(ids[0]						,						__a )
          self.assertEqual(len(__a )						,						__a )






      def lowercase_							(					self				:    str ):


          '''simple docstring'''


          self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR'''] )						,						[256_203, 3] )






      def lowercase_							(					self				:    str ):


          '''simple docstring'''


          UpperCAmelCase__			:		List[str]													=				tempfile.mkdtemp()
          UpperCAmelCase__			:		Union[str, Any]													=				self.tokenizer.fairseq_tokens_to_ids
          self.tokenizer.save_pretrained(__a )
          UpperCAmelCase__			:		Union[str, Any]													=				NllbTokenizer.from_pretrained(__a )
          self.assertDictEqual(new_tok.fairseq_tokens_to_ids						,						__a )






      @require_torch
      def lowercase_							(					self				:    str ):


          '''simple docstring'''


          UpperCAmelCase__			:		Optional[Any]													=				self.tokenizer(
              self.src_text						,						text_target=self.tgt_text						,						padding=__a						,						truncation=__a						,						max_length=len(self.expected_src_tokens )						,						return_tensors='''pt'''						,						)
          UpperCAmelCase__			:		List[Any]													=				shift_tokens_right(
              batch['''labels''']						,						self.tokenizer.pad_token_id						,						self.tokenizer.lang_code_to_id['''ron_Latn'''] )

          self.assertIsInstance(__a						,						__a )

          self.assertEqual((2, 15)						,						batch.input_ids.shape )
          self.assertEqual((2, 15)						,						batch.attention_mask.shape )
          UpperCAmelCase__			:		Union[str, Any]													=				batch.input_ids.tolist()[0]
          self.assertListEqual(self.expected_src_tokens						,						__a )
          self.assertEqual(__a						,						batch.decoder_input_ids[0, 0] )  # EOS
          # Test that special tokens are reset
          self.assertEqual(self.tokenizer.prefix_tokens						,						[EN_CODE] )
          self.assertEqual(self.tokenizer.suffix_tokens						,						[self.tokenizer.eos_token_id] )






      def lowercase_							(					self				:    Union[str, Any] ):


          '''simple docstring'''


          UpperCAmelCase__			:		List[Any]													=				self.tokenizer(self.src_text						,						padding=__a						,						truncation=__a						,						max_length=3						,						return_tensors='''pt''' )
          UpperCAmelCase__			:		Dict													=				self.tokenizer(
              text_target=self.tgt_text						,						padding=__a						,						truncation=__a						,						max_length=10						,						return_tensors='''pt''' )
          UpperCAmelCase__			:		Union[str, Any]													=				targets['''input_ids''']
          UpperCAmelCase__			:		Optional[int]													=				shift_tokens_right(
              __a						,						self.tokenizer.pad_token_id						,						decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]						,						)

          self.assertEqual(batch.input_ids.shape[1]						,						3 )
          self.assertEqual(batch.decoder_input_ids.shape[1]						,						10 )






      @require_torch
      def lowercase_							(					self				:    List[str] ):


          '''simple docstring'''


          UpperCAmelCase__			:		str													=				self.tokenizer._build_translation_inputs(
              '''A test'''						,						return_tensors='''pt'''						,						src_lang='''eng_Latn'''						,						tgt_lang='''fra_Latn''' )

          self.assertEqual(
              nested_simplify(__a )						,						{
                  # A, test, EOS, en_XX
                  '''input_ids''': [[256_047, 70, 7_356, 2]],
                  '''attention_mask''': [[1, 1, 1, 1]],
                  # ar_AR
                  '''forced_bos_token_id''': 256_057,
              }						,						)






      @require_torch
      def lowercase_							(					self				:    Optional[int] ):


          '''simple docstring'''


          UpperCAmelCase__			:		List[Any]													=				True
          UpperCAmelCase__			:		Union[str, Any]													=				self.tokenizer(
              '''UN Chief says there is no military solution in Syria'''						,						src_lang='''eng_Latn'''						,						tgt_lang='''fra_Latn''' )
          self.assertEqual(
              inputs.input_ids						,						[16_297, 134_408, 25_653, 6_370, 248, 254, 103_929, 94_995, 108, 49_486, 2, 256_047] )

          UpperCAmelCase__			:		Union[str, Any]													=				False
          UpperCAmelCase__			:		Any													=				self.tokenizer(
              '''UN Chief says there is no military solution in Syria'''						,						src_lang='''eng_Latn'''						,						tgt_lang='''fra_Latn''' )
          self.assertEqual(
              inputs.input_ids						,						[256_047, 16_297, 134_408, 25_653, 6_370, 248, 254, 103_929, 94_995, 108, 49_486, 2] )




"""simple docstring"""


from diffusers.utils.testing_utils import require_onnxruntime
@require_onnxruntime
class   _UpperCamelCase  :
							'''simple docstring'''







							pass




