


'''simple docstring'''






import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


a_    :			Optional[int]      				=			get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
 from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

a_    :			List[str]      				=			2_5_6_0_4_7
a_    :			Any      				=			2_5_6_1_4_5




@require_sentencepiece
@require_tokenizers
class        a   (   snake_case_     , unittest.TestCase				):
 _lowerCAmelCase           =		NllbTokenizer
 _lowerCAmelCase           =		NllbTokenizerFast
 _lowerCAmelCase           =		True
 _lowerCAmelCase           =		True
 _lowerCAmelCase           =		{}
 def 							__UpperCAmelCase						(       self  )    ->							Dict:
      super().setUp()

      # We have a SentencePiece fixture for testing
      _a						=							NllbTokenizer(__magic_name__					, keep_accents=__magic_name__  )
      tokenizer.save_pretrained(self.tmpdirname  )
 def 							__UpperCAmelCase						(       self  )    ->							Optional[int]:
      _a						=							NllbTokenizer(__magic_name__					, keep_accents=__magic_name__  )

      _a						=							tokenizer.tokenize('This is a test'  )
      self.assertListEqual(__magic_name__					, ['▁This', '▁is', '▁a', '▁t', 'est']  )

      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(__magic_name__  )					, [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]					, )

      _a						=							tokenizer.tokenize('I was born in 92000, and this is falsé.'  )
      self.assertListEqual(
          __magic_name__					, [
              SPIECE_UNDERLINE + 'I',
              SPIECE_UNDERLINE + 'was',
              SPIECE_UNDERLINE + 'b',
              'or',
              'n',
              SPIECE_UNDERLINE + 'in',
              SPIECE_UNDERLINE + '',
              '9',
              '2',
              '0',
              '0',
              '0',
              ',',
              SPIECE_UNDERLINE + 'and',
              SPIECE_UNDERLINE + 'this',
              SPIECE_UNDERLINE + 'is',
              SPIECE_UNDERLINE + 'f',
              'al',
              's',
              'é',
              '.',
          ]					, )
      _a						=							tokenizer.convert_tokens_to_ids(__magic_name__  )
      self.assertListEqual(
          __magic_name__					, [
              value + tokenizer.fairseq_offset
              for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
          ]					, )

      _a						=							tokenizer.convert_ids_to_tokens(__magic_name__  )
      self.assertListEqual(
          __magic_name__					, [
              SPIECE_UNDERLINE + 'I',
              SPIECE_UNDERLINE + 'was',
              SPIECE_UNDERLINE + 'b',
              'or',
              'n',
              SPIECE_UNDERLINE + 'in',
              SPIECE_UNDERLINE + '',
              '<unk>',
              '2',
              '0',
              '0',
              '0',
              ',',
              SPIECE_UNDERLINE + 'and',
              SPIECE_UNDERLINE + 'this',
              SPIECE_UNDERLINE + 'is',
              SPIECE_UNDERLINE + 'f',
              'al',
              's',
              '<unk>',
              '.',
          ]					, )
 def 							__UpperCAmelCase						(       self  )    ->							List[str]:
      _a						=							(self.rust_tokenizer_class, 'hf-internal-testing/tiny-random-nllb', {})
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'  ):
                _a						=							self.rust_tokenizer_class.from_pretrained(__magic_name__					, **__magic_name__  )
                _a						=							self.tokenizer_class.from_pretrained(__magic_name__					, **__magic_name__  )

                _a						=							tempfile.mkdtemp()

                _a						=							tokenizer_r.save_pretrained(__magic_name__  )
                _a						=							tokenizer_p.save_pretrained(__magic_name__  )

                # Checks it save with the same files + the tokenizer.json file for the fast one
                self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files  )  )
                _a						=							tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f  )
                self.assertSequenceEqual(__magic_name__					, __magic_name__  )

                # Checks everything loads correctly in the same way
                _a						=							tokenizer_r.from_pretrained(__magic_name__  )
                _a						=							tokenizer_p.from_pretrained(__magic_name__  )

                # Check special tokens are set accordingly on Rust and Python
                for key in tokenizer_pp.special_tokens_map:
                     self.assertTrue(hasattr(__magic_name__					, __magic_name__  )  )

                shutil.rmtree(__magic_name__  )

                # Save tokenizer rust, legacy_format=True
                _a						=							tempfile.mkdtemp()

                _a						=							tokenizer_r.save_pretrained(__magic_name__					, legacy_format=__magic_name__  )
                _a						=							tokenizer_p.save_pretrained(__magic_name__  )

                # Checks it save with the same files
                self.assertSequenceEqual(__magic_name__					, __magic_name__  )

                # Checks everything loads correctly in the same way
                _a						=							tokenizer_r.from_pretrained(__magic_name__  )
                _a						=							tokenizer_p.from_pretrained(__magic_name__  )

                # Check special tokens are set accordingly on Rust and Python
                for key in tokenizer_pp.special_tokens_map:
                     self.assertTrue(hasattr(__magic_name__					, __magic_name__  )  )

                shutil.rmtree(__magic_name__  )

                # Save tokenizer rust, legacy_format=False
                _a						=							tempfile.mkdtemp()

                _a						=							tokenizer_r.save_pretrained(__magic_name__					, legacy_format=__magic_name__  )
                _a						=							tokenizer_p.save_pretrained(__magic_name__  )

                # Checks it saved the tokenizer.json file
                self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files  )  )

                # Checks everything loads correctly in the same way
                _a						=							tokenizer_r.from_pretrained(__magic_name__  )
                _a						=							tokenizer_p.from_pretrained(__magic_name__  )

                # Check special tokens are set accordingly on Rust and Python
                for key in tokenizer_pp.special_tokens_map:
                     self.assertTrue(hasattr(__magic_name__					, __magic_name__  )  )

                shutil.rmtree(__magic_name__  )
 @require_torch
 def 							__UpperCAmelCase						(       self  )    ->							str:
      if not self.test_seqaseq:
           return

      _a						=							self.get_tokenizers()
      for tokenizer in tokenizers:
           with self.subTest(f'{tokenizer.__class__.__name__}'  ):
                # Longer text that will definitely require truncation.
                _a						=							[
                    ' UN Chief Says There Is No Military Solution in Syria',
                    ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'
                    ' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'
                    ' will only worsen the violence and misery for millions of people.',
                ]
                _a						=							[
                    'Şeful ONU declară că nu există o soluţie militară în Siria',
                    'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'
                    ' Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi'
                    ' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
                ]
                try:
                     _a						=							tokenizer.prepare_seqaseq_batch(
                         src_texts=__magic_name__					, tgt_texts=__magic_name__					, max_length=3					, max_target_length=10					, return_tensors='pt'					, src_lang='eng_Latn'					, tgt_lang='ron_Latn'					, )
                except NotImplementedError:
                     return
                self.assertEqual(batch.input_ids.shape[1]					, 3  )
                self.assertEqual(batch.labels.shape[1]					, 10  )
                # max_target_length will default to max_length if not specified
                _a						=							tokenizer.prepare_seqaseq_batch(
                    __magic_name__					, tgt_texts=__magic_name__					, max_length=3					, return_tensors='pt'  )
                self.assertEqual(batch.input_ids.shape[1]					, 3  )
                self.assertEqual(batch.labels.shape[1]					, 3  )

                _a						=							tokenizer.prepare_seqaseq_batch(
                    src_texts=__magic_name__					, max_length=3					, max_target_length=10					, return_tensors='pt'  )
                self.assertEqual(batch_encoder_only.input_ids.shape[1]					, 3  )
                self.assertEqual(batch_encoder_only.attention_mask.shape[1]					, 3  )
                self.assertNotIn('decoder_input_ids'					, __magic_name__  )
 @unittest.skip('Unfortunately way too slow to build a BPE with SentencePiece.'  )
 def 							__UpperCAmelCase						(       self  )    ->							Any:
      pass




 def 							__UpperCAmelCase						(       self  )    ->							List[Any]:
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'  ):
                _a						=							[AddedToken('<special>'					, lstrip=__magic_name__  )]

                _a						=							self.rust_tokenizer_class.from_pretrained(
                    __magic_name__					, additional_special_tokens=__magic_name__					, **__magic_name__  )
                _a						=							tokenizer_r.encode('Hey this is a <special> token'  )

                _a						=							tokenizer_r.encode('<special>'					, add_special_tokens=__magic_name__  )[0]

                self.assertTrue(special_token_id in r_output  )

                if self.test_slow_tokenizer:
                     _a						=							self.rust_tokenizer_class.from_pretrained(
                         __magic_name__					, additional_special_tokens=__magic_name__					, **__magic_name__					, )
                     _a						=							self.tokenizer_class.from_pretrained(
                         __magic_name__					, additional_special_tokens=__magic_name__					, **__magic_name__  )

                     _a						=							tokenizer_p.encode('Hey this is a <special> token'  )

                     _a						=							tokenizer_cr.encode('Hey this is a <special> token'  )

                     self.assertEqual(__magic_name__					, __magic_name__  )
                     self.assertEqual(__magic_name__					, __magic_name__  )
                     self.assertTrue(special_token_id in p_output  )
                     self.assertTrue(special_token_id in cr_output  )





@require_torch
@require_sentencepiece
@require_tokenizers
class        a   (   unittest.TestCase				):
 _lowerCAmelCase           =		"""facebook/nllb-200-distilled-600M"""
 _lowerCAmelCase           =		[
     """ UN Chief Says There Is No Military Solution in Syria""",
     """ Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
 ]
 _lowerCAmelCase           =		[
     """Şeful ONU declară că nu există o soluţie militară în Siria""",
     """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"""
     """ pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"""
     """ face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
 ]
 _lowerCAmelCase           =		[
     2_5_6_0_4_7,
     1_6_2_9_7,
     1_3_4_4_0_8,
     8_1_6_5,
     2_4_8_0_6_6,
     1_4_7_3_4,
     9_5_0,
     1_1_3_5,
     1_0_5_7_2_1,
     3_5_7_3,
     8_3,
     2_7_3_5_2,
     1_0_8,
     4_9_4_8_6,
     2,
 ]
 @classmethod
 def 							__UpperCAmelCase						(       cls  )    ->							int:
      _a						=							NllbTokenizer.from_pretrained(
          cls.checkpoint_name					, src_lang='eng_Latn'					, tgt_lang='ron_Latn'  )
      _a						=							1
      return cls
 def 							__UpperCAmelCase						(       self  )    ->							Optional[int]:
      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Arab']					, 25_60_01  )
      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Latn']					, 25_60_02  )
      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['fra_Latn']					, 25_60_57  )
 def 							__UpperCAmelCase						(       self  )    ->							Any:
      _a						=							self.tokenizer.batch_encode_plus(self.src_text  ).input_ids[0]
      self.assertListEqual(self.expected_src_tokens					, __magic_name__  )
 def 							__UpperCAmelCase						(       self  )    ->							Optional[Any]:
      self.assertIn(__magic_name__					, self.tokenizer.all_special_ids  )
      # fmt: off
      _a						=							[RO_CODE, 42_54, 9_80_68, 11_29_23, 3_90_72, 39_09, 7_13, 10_27_67, 26, 1_73_14, 3_56_42, 1_46_83, 3_31_18, 20_22, 6_69_87, 2, 25_60_47]
      # fmt: on

      _a						=							self.tokenizer.decode(__magic_name__					, skip_special_tokens=__magic_name__  )
      _a						=							self.tokenizer.decode(generated_ids[1:]					, skip_special_tokens=__magic_name__  )
      self.assertEqual(__magic_name__					, __magic_name__  )
      self.assertNotIn(self.tokenizer.eos_token					, __magic_name__  )
 def 							__UpperCAmelCase						(       self  )    ->							Any:
      _a						=							['this is gunna be a long sentence ' * 20]
      assert isinstance(src_text[0]					, __magic_name__  )
      _a						=							10
      _a						=							self.tokenizer(__magic_name__					, max_length=__magic_name__					, truncation=__magic_name__  ).input_ids[0]
      self.assertEqual(ids[-1]					, 2  )
      self.assertEqual(ids[0]					, __magic_name__  )
      self.assertEqual(len(__magic_name__  )					, __magic_name__  )
 def 							__UpperCAmelCase						(       self  )    ->							Any:
      self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR']  )					, [25_62_03, 3]  )
 def 							__UpperCAmelCase						(       self  )    ->							Optional[int]:
      _a						=							tempfile.mkdtemp()
      _a						=							self.tokenizer.fairseq_tokens_to_ids
      self.tokenizer.save_pretrained(__magic_name__  )
      _a						=							NllbTokenizer.from_pretrained(__magic_name__  )
      self.assertDictEqual(new_tok.fairseq_tokens_to_ids					, __magic_name__  )
 @require_torch
 def 							__UpperCAmelCase						(       self  )    ->							int:
      _a						=							self.tokenizer(
          self.src_text					, text_target=self.tgt_text					, padding=__magic_name__					, truncation=__magic_name__					, max_length=len(self.expected_src_tokens  )					, return_tensors='pt'					, )
      _a						=							shift_tokens_right(
          batch['labels']					, self.tokenizer.pad_token_id					, self.tokenizer.lang_code_to_id['ron_Latn']  )

      self.assertIsInstance(__magic_name__					, __magic_name__  )

      self.assertEqual((2, 15)					, batch.input_ids.shape  )
      self.assertEqual((2, 15)					, batch.attention_mask.shape  )
      _a						=							batch.input_ids.tolist()[0]
      self.assertListEqual(self.expected_src_tokens					, __magic_name__  )
      self.assertEqual(__magic_name__					, batch.decoder_input_ids[0, 0]  )  # EOS
      # Test that special tokens are reset
      self.assertEqual(self.tokenizer.prefix_tokens					, [EN_CODE]  )
      self.assertEqual(self.tokenizer.suffix_tokens					, [self.tokenizer.eos_token_id]  )
 def 							__UpperCAmelCase						(       self  )    ->							int:
      _a						=							self.tokenizer(self.src_text					, padding=__magic_name__					, truncation=__magic_name__					, max_length=3					, return_tensors='pt'  )
      _a						=							self.tokenizer(
          text_target=self.tgt_text					, padding=__magic_name__					, truncation=__magic_name__					, max_length=10					, return_tensors='pt'  )
      _a						=							targets['input_ids']
      _a						=							shift_tokens_right(
          __magic_name__					, self.tokenizer.pad_token_id					, decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]					, )

      self.assertEqual(batch.input_ids.shape[1]					, 3  )
      self.assertEqual(batch.decoder_input_ids.shape[1]					, 10  )
 @require_torch
 def 							__UpperCAmelCase						(       self  )    ->							List[Any]:
      _a						=							self.tokenizer._build_translation_inputs(
          'A test'					, return_tensors='pt'					, src_lang='eng_Latn'					, tgt_lang='fra_Latn'  )

      self.assertEqual(
          nested_simplify(__magic_name__  )					, {
              # A, test, EOS, en_XX
              'input_ids': [[25_60_47, 70, 73_56, 2]],
              'attention_mask': [[1, 1, 1, 1]],
              # ar_AR
              'forced_bos_token_id': 25_60_57,
          }					, )




 @require_torch
 def 							__UpperCAmelCase						(       self  )    ->							Tuple:
      _a						=							True
      _a						=							self.tokenizer(
          'UN Chief says there is no military solution in Syria'					, src_lang='eng_Latn'					, tgt_lang='fra_Latn'  )
      self.assertEqual(
          inputs.input_ids					, [1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2, 25_60_47]  )

      _a						=							False
      _a						=							self.tokenizer(
          'UN Chief says there is no military solution in Syria'					, src_lang='eng_Latn'					, tgt_lang='fra_Latn'  )
      self.assertEqual(
          inputs.input_ids					, [25_60_47, 1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2]  )



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__A				={
    '''configuration_bloom''': ['''BLOOM_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''BloomConfig''', '''BloomOnnxConfig'''],
}
try:
			if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			__A				=['''BloomTokenizerFast''']

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			__A				=[
			    '''BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST''',
			    '''BloomForCausalLM''',
			    '''BloomModel''',
			    '''BloomPreTrainedModel''',
			    '''BloomForSequenceClassification''',
			    '''BloomForTokenClassification''',
			    '''BloomForQuestionAnswering''',
			]

if TYPE_CHECKING:
			from .configuration_bloom import BLOOM_PRETRAINED_CONFIG_ARCHIVE_MAP, BloomConfig, BloomOnnxConfig

			try:
						if not is_tokenizers_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_bloom_fast import BloomTokenizerFast

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_bloom import (
						    BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST,
						    BloomForCausalLM,
						    BloomForQuestionAnswering,
						    BloomForSequenceClassification,
						    BloomForTokenClassification,
						    BloomModel,
						    BloomPreTrainedModel,
						)

else:
			import sys

			__A				=_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)


