

import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase_       	=	get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
					from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

lowerCAmelCase_       	=	25_60_47
lowerCAmelCase_       	=	25_61_45





@require_sentencepiece
@require_tokenizers
class 	snake_case_   (		__A	,       unittest.TestCase					):







			'''simple docstring'''

			SCREAMING_SNAKE_CASE     :   Tuple									=				NllbTokenizer
			SCREAMING_SNAKE_CASE     :   Any									=				NllbTokenizerFast
			SCREAMING_SNAKE_CASE     :   str									=				True
			SCREAMING_SNAKE_CASE     :   List[str]									=				True
			SCREAMING_SNAKE_CASE     :   Union[str, Any]									=				{}







			def 				snake_case__(     self :				List[str]							)      ->Union[str, Any]:
									super().setUp()

									# We have a SentencePiece fixture for testing
									snake_case_   =	NllbTokenizer(_UpperCamelCase							,   keep_accents=_UpperCamelCase							)
									tokenizer.save_pretrained(self.tmpdirname							)







			def 				snake_case__(     self :				List[str]							)      ->int:
									snake_case_   =	NllbTokenizer(_UpperCamelCase							,   keep_accents=_UpperCamelCase							)

									snake_case_   =	tokenizer.tokenize('''This is a test'''							)
									self.assertListEqual(_UpperCamelCase							,   ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']							)

									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(_UpperCamelCase							)							,   [value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]							,   )

									snake_case_   =	tokenizer.tokenize('''I was born in 92000, and this is falsé.'''							)
									self.assertListEqual(
									    _UpperCamelCase							,   [
									        SPIECE_UNDERLINE + '''I''',
									        SPIECE_UNDERLINE + '''was''',
									        SPIECE_UNDERLINE + '''b''',
									        '''or''',
									        '''n''',
									        SPIECE_UNDERLINE + '''in''',
									        SPIECE_UNDERLINE + '''''',
									        '''9''',
									        '''2''',
									        '''0''',
									        '''0''',
									        '''0''',
									        ''',''',
									        SPIECE_UNDERLINE + '''and''',
									        SPIECE_UNDERLINE + '''this''',
									        SPIECE_UNDERLINE + '''is''',
									        SPIECE_UNDERLINE + '''f''',
									        '''al''',
									        '''s''',
									        '''é''',
									        '''.''',
									    ]							,   )
									snake_case_   =	tokenizer.convert_tokens_to_ids(_UpperCamelCase							)
									self.assertListEqual(
									    _UpperCamelCase							,   [
									        value + tokenizer.fairseq_offset
									        for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
									    ]							,   )

									snake_case_   =	tokenizer.convert_ids_to_tokens(_UpperCamelCase							)
									self.assertListEqual(
									    _UpperCamelCase							,   [
									        SPIECE_UNDERLINE + '''I''',
									        SPIECE_UNDERLINE + '''was''',
									        SPIECE_UNDERLINE + '''b''',
									        '''or''',
									        '''n''',
									        SPIECE_UNDERLINE + '''in''',
									        SPIECE_UNDERLINE + '''''',
									        '''<unk>''',
									        '''2''',
									        '''0''',
									        '''0''',
									        '''0''',
									        ''',''',
									        SPIECE_UNDERLINE + '''and''',
									        SPIECE_UNDERLINE + '''this''',
									        SPIECE_UNDERLINE + '''is''',
									        SPIECE_UNDERLINE + '''f''',
									        '''al''',
									        '''s''',
									        '''<unk>''',
									        '''.''',
									    ]							,   )







			def 				snake_case__(     self :				List[str]							)      ->Any:
									snake_case_   =	(self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
																					snake_case_   =	self.rust_tokenizer_class.from_pretrained(_UpperCamelCase							,   **_UpperCamelCase							)
																					snake_case_   =	self.tokenizer_class.from_pretrained(_UpperCamelCase							,   **_UpperCamelCase							)

																					snake_case_   =	tempfile.mkdtemp()

																					snake_case_   =	tokenizer_r.save_pretrained(_UpperCamelCase							)
																					snake_case_   =	tokenizer_p.save_pretrained(_UpperCamelCase							)

																					# Checks it save with the same files + the tokenizer.json file for the fast one
																					self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files							)							)
																					snake_case_   =	tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f							)
																					self.assertSequenceEqual(_UpperCamelCase							,   _UpperCamelCase							)

																					# Checks everything loads correctly in the same way
																					snake_case_   =	tokenizer_r.from_pretrained(_UpperCamelCase							)
																					snake_case_   =	tokenizer_p.from_pretrained(_UpperCamelCase							)

																					# Check special tokens are set accordingly on Rust and Python
																					for key in tokenizer_pp.special_tokens_map:
																											self.assertTrue(hasattr(_UpperCamelCase							,   _UpperCamelCase							)							)

																					shutil.rmtree(_UpperCamelCase							)

																					# Save tokenizer rust, legacy_format=True
																					snake_case_   =	tempfile.mkdtemp()

																					snake_case_   =	tokenizer_r.save_pretrained(_UpperCamelCase							,   legacy_format=_UpperCamelCase							)
																					snake_case_   =	tokenizer_p.save_pretrained(_UpperCamelCase							)

																					# Checks it save with the same files
																					self.assertSequenceEqual(_UpperCamelCase							,   _UpperCamelCase							)

																					# Checks everything loads correctly in the same way
																					snake_case_   =	tokenizer_r.from_pretrained(_UpperCamelCase							)
																					snake_case_   =	tokenizer_p.from_pretrained(_UpperCamelCase							)

																					# Check special tokens are set accordingly on Rust and Python
																					for key in tokenizer_pp.special_tokens_map:
																											self.assertTrue(hasattr(_UpperCamelCase							,   _UpperCamelCase							)							)

																					shutil.rmtree(_UpperCamelCase							)

																					# Save tokenizer rust, legacy_format=False
																					snake_case_   =	tempfile.mkdtemp()

																					snake_case_   =	tokenizer_r.save_pretrained(_UpperCamelCase							,   legacy_format=_UpperCamelCase							)
																					snake_case_   =	tokenizer_p.save_pretrained(_UpperCamelCase							)

																					# Checks it saved the tokenizer.json file
																					self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files							)							)

																					# Checks everything loads correctly in the same way
																					snake_case_   =	tokenizer_r.from_pretrained(_UpperCamelCase							)
																					snake_case_   =	tokenizer_p.from_pretrained(_UpperCamelCase							)

																					# Check special tokens are set accordingly on Rust and Python
																					for key in tokenizer_pp.special_tokens_map:
																											self.assertTrue(hasattr(_UpperCamelCase							,   _UpperCamelCase							)							)

																					shutil.rmtree(_UpperCamelCase							)







			@require_torch
			def 				snake_case__(     self :				Any							)      ->Optional[Any]:
									if not self.test_seqaseq:
															return

									snake_case_   =	self.get_tokenizers()
									for tokenizer in tokenizers:
															with self.subTest(f'''{tokenizer.__class__.__name__}'''							):
																					# Longer text that will definitely require truncation.
																					snake_case_   =	[
																					    ''' UN Chief Says There Is No Military Solution in Syria''',
																					    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
																					    ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
																					    ''' will only worsen the violence and misery for millions of people.''',
																					]
																					snake_case_   =	[
																					    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
																					    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
																					    ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
																					    ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
																					]
																					try:
																											snake_case_   =	tokenizer.prepare_seqaseq_batch(
																											    src_texts=_UpperCamelCase							,   tgt_texts=_UpperCamelCase							,   max_length=3							,   max_target_length=1_0							,   return_tensors='''pt'''							,   src_lang='''eng_Latn'''							,   tgt_lang='''ron_Latn'''							,   )
																					except NotImplementedError:
																											return
																					self.assertEqual(batch.input_ids.shape[1]							,   3							)
																					self.assertEqual(batch.labels.shape[1]							,   1_0							)
																					# max_target_length will default to max_length if not specified
																					snake_case_   =	tokenizer.prepare_seqaseq_batch(
																					    _UpperCamelCase							,   tgt_texts=_UpperCamelCase							,   max_length=3							,   return_tensors='''pt'''							)
																					self.assertEqual(batch.input_ids.shape[1]							,   3							)
																					self.assertEqual(batch.labels.shape[1]							,   3							)

																					snake_case_   =	tokenizer.prepare_seqaseq_batch(
																					    src_texts=_UpperCamelCase							,   max_length=3							,   max_target_length=1_0							,   return_tensors='''pt'''							)
																					self.assertEqual(batch_encoder_only.input_ids.shape[1]							,   3							)
																					self.assertEqual(batch_encoder_only.attention_mask.shape[1]							,   3							)
																					self.assertNotIn('''decoder_input_ids'''							,   _UpperCamelCase							)







			@unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.'''							)
			def 				snake_case__(     self :				List[str]							)      ->Tuple:
									pass







			def 				snake_case__(     self :				Any							)      ->Any:
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
																					snake_case_   =	[AddedToken('''<special>'''							,   lstrip=_UpperCamelCase							)]

																					snake_case_   =	self.rust_tokenizer_class.from_pretrained(
																					    _UpperCamelCase							,   additional_special_tokens=_UpperCamelCase							,   **_UpperCamelCase							)
																					snake_case_   =	tokenizer_r.encode('''Hey this is a <special> token'''							)

																					snake_case_   =	tokenizer_r.encode('''<special>'''							,   add_special_tokens=_UpperCamelCase							)[0]

																					self.assertTrue(special_token_id in r_output							)

																					if self.test_slow_tokenizer:
																											snake_case_   =	self.rust_tokenizer_class.from_pretrained(
																											    _UpperCamelCase							,   additional_special_tokens=_UpperCamelCase							,   **_UpperCamelCase							,   )
																											snake_case_   =	self.tokenizer_class.from_pretrained(
																											    _UpperCamelCase							,   additional_special_tokens=_UpperCamelCase							,   **_UpperCamelCase							)

																											snake_case_   =	tokenizer_p.encode('''Hey this is a <special> token'''							)

																											snake_case_   =	tokenizer_cr.encode('''Hey this is a <special> token'''							)

																											self.assertEqual(_UpperCamelCase							,   _UpperCamelCase							)
																											self.assertEqual(_UpperCamelCase							,   _UpperCamelCase							)
																											self.assertTrue(special_token_id in p_output							)
																											self.assertTrue(special_token_id in cr_output							)






@require_torch
@require_sentencepiece
@require_tokenizers
class 	snake_case_   (		unittest.TestCase					):







			'''simple docstring'''

			SCREAMING_SNAKE_CASE     :   Any									=				"facebook/nllb-200-distilled-600M"
			SCREAMING_SNAKE_CASE     :   int									=				[
			    " UN Chief Says There Is No Military Solution in Syria",
			    " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
			]
			SCREAMING_SNAKE_CASE     :   Dict									=				[
			    "Şeful ONU declară că nu există o soluţie militară în Siria",
			    "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
			    " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
			    " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
			]
			SCREAMING_SNAKE_CASE     :   Dict									=				[
			    256047,
			    16297,
			    134408,
			    8165,
			    248066,
			    14734,
			    950,
			    1135,
			    105721,
			    3573,
			    83,
			    27352,
			    108,
			    49486,
			    2,
			]







			@classmethod
			def 				snake_case__(     cls :				Optional[int]							)      ->int:
									snake_case_   =	NllbTokenizer.from_pretrained(
									    cls.checkpoint_name							,   src_lang='''eng_Latn'''							,   tgt_lang='''ron_Latn'''							)
									snake_case_   =	1
									return cls







			def 				snake_case__(     self :				Optional[int]							)      ->Any:
									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab''']							,   2_5_6_0_0_1							)
									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn''']							,   2_5_6_0_0_2							)
									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn''']							,   2_5_6_0_5_7							)







			def 				snake_case__(     self :				str							)      ->Tuple:
									snake_case_   =	self.tokenizer.batch_encode_plus(self.src_text							).input_ids[0]
									self.assertListEqual(self.expected_src_tokens							,   _UpperCamelCase							)







			def 				snake_case__(     self :				Any							)      ->List[str]:
									self.assertIn(_UpperCamelCase							,   self.tokenizer.all_special_ids							)
									# fmt: off
									snake_case_   =	[RO_CODE, 4_2_5_4, 9_8_0_6_8, 1_1_2_9_2_3, 3_9_0_7_2, 3_9_0_9, 7_1_3, 1_0_2_7_6_7, 2_6, 1_7_3_1_4, 3_5_6_4_2, 1_4_6_8_3, 3_3_1_1_8, 2_0_2_2, 6_6_9_8_7, 2, 2_5_6_0_4_7]
									# fmt: on

									snake_case_   =	self.tokenizer.decode(_UpperCamelCase							,   skip_special_tokens=_UpperCamelCase							)
									snake_case_   =	self.tokenizer.decode(generated_ids[1:]							,   skip_special_tokens=_UpperCamelCase							)
									self.assertEqual(_UpperCamelCase							,   _UpperCamelCase							)
									self.assertNotIn(self.tokenizer.eos_token							,   _UpperCamelCase							)







			def 				snake_case__(     self :				Dict							)      ->str:
									snake_case_   =	['''this is gunna be a long sentence ''' * 2_0]
									assert isinstance(src_text[0]							,   _UpperCamelCase							)
									snake_case_   =	1_0
									snake_case_   =	self.tokenizer(_UpperCamelCase							,   max_length=_UpperCamelCase							,   truncation=_UpperCamelCase							).input_ids[0]
									self.assertEqual(ids[-1]							,   2							)
									self.assertEqual(ids[0]							,   _UpperCamelCase							)
									self.assertEqual(len(_UpperCamelCase							)							,   _UpperCamelCase							)







			def 				snake_case__(     self :				Tuple							)      ->Union[str, Any]:
									self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']							)							,   [2_5_6_2_0_3, 3]							)







			def 				snake_case__(     self :				Dict							)      ->List[Any]:
									snake_case_   =	tempfile.mkdtemp()
									snake_case_   =	self.tokenizer.fairseq_tokens_to_ids
									self.tokenizer.save_pretrained(_UpperCamelCase							)
									snake_case_   =	NllbTokenizer.from_pretrained(_UpperCamelCase							)
									self.assertDictEqual(new_tok.fairseq_tokens_to_ids							,   _UpperCamelCase							)







			@require_torch
			def 				snake_case__(     self :				str							)      ->Dict:
									snake_case_   =	self.tokenizer(
									    self.src_text							,   text_target=self.tgt_text							,   padding=_UpperCamelCase							,   truncation=_UpperCamelCase							,   max_length=len(self.expected_src_tokens							)							,   return_tensors='''pt'''							,   )
									snake_case_   =	shift_tokens_right(
									    batch['''labels''']							,   self.tokenizer.pad_token_id							,   self.tokenizer.lang_code_to_id['''ron_Latn''']							)

									self.assertIsInstance(_UpperCamelCase							,   _UpperCamelCase							)

									self.assertEqual((2, 1_5)							,   batch.input_ids.shape							)
									self.assertEqual((2, 1_5)							,   batch.attention_mask.shape							)
									snake_case_   =	batch.input_ids.tolist()[0]
									self.assertListEqual(self.expected_src_tokens							,   _UpperCamelCase							)
									self.assertEqual(_UpperCamelCase							,   batch.decoder_input_ids[0, 0]							)  # EOS
									# Test that special tokens are reset
									self.assertEqual(self.tokenizer.prefix_tokens							,   [EN_CODE]							)
									self.assertEqual(self.tokenizer.suffix_tokens							,   [self.tokenizer.eos_token_id]							)







			def 				snake_case__(     self :				List[Any]							)      ->Any:
									snake_case_   =	self.tokenizer(self.src_text							,   padding=_UpperCamelCase							,   truncation=_UpperCamelCase							,   max_length=3							,   return_tensors='''pt'''							)
									snake_case_   =	self.tokenizer(
									    text_target=self.tgt_text							,   padding=_UpperCamelCase							,   truncation=_UpperCamelCase							,   max_length=1_0							,   return_tensors='''pt'''							)
									snake_case_   =	targets['''input_ids''']
									snake_case_   =	shift_tokens_right(
									    _UpperCamelCase							,   self.tokenizer.pad_token_id							,   decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]							,   )

									self.assertEqual(batch.input_ids.shape[1]							,   3							)
									self.assertEqual(batch.decoder_input_ids.shape[1]							,   1_0							)







			@require_torch
			def 				snake_case__(     self :				Optional[int]							)      ->Union[str, Any]:
									snake_case_   =	self.tokenizer._build_translation_inputs(
									    '''A test'''							,   return_tensors='''pt'''							,   src_lang='''eng_Latn'''							,   tgt_lang='''fra_Latn'''							)

									self.assertEqual(
									    nested_simplify(_UpperCamelCase							)							,   {
									        # A, test, EOS, en_XX
									        '''input_ids''': [[2_5_6_0_4_7, 7_0, 7_3_5_6, 2]],
									        '''attention_mask''': [[1, 1, 1, 1]],
									        # ar_AR
									        '''forced_bos_token_id''': 2_5_6_0_5_7,
									    }							,   )







			@require_torch
			def 				snake_case__(     self :				List[str]							)      ->str:
									snake_case_   =	True
									snake_case_   =	self.tokenizer(
									    '''UN Chief says there is no military solution in Syria'''							,   src_lang='''eng_Latn'''							,   tgt_lang='''fra_Latn'''							)
									self.assertEqual(
									    inputs.input_ids							,   [1_6_2_9_7, 1_3_4_4_0_8, 2_5_6_5_3, 6_3_7_0, 2_4_8, 2_5_4, 1_0_3_9_2_9, 9_4_9_9_5, 1_0_8, 4_9_4_8_6, 2, 2_5_6_0_4_7]							)

									snake_case_   =	False
									snake_case_   =	self.tokenizer(
									    '''UN Chief says there is no military solution in Syria'''							,   src_lang='''eng_Latn'''							,   tgt_lang='''fra_Latn'''							)
									self.assertEqual(
									    inputs.input_ids							,   [2_5_6_0_4_7, 1_6_2_9_7, 1_3_4_4_0_8, 2_5_6_5_3, 6_3_7_0, 2_4_8, 2_5_4, 1_0_3_9_2_9, 9_4_9_9_5, 1_0_8, 4_9_4_8_6, 2]							)


# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ..models.auto import AutoModelForSeqaSeqLM, AutoTokenizer
from .base import PipelineTool





class 	snake_case_   (		__A					):







			'''simple docstring'''

			SCREAMING_SNAKE_CASE     :   Optional[Any]									=				"philschmid/bart-large-cnn-samsum"
			SCREAMING_SNAKE_CASE     :   Tuple									=				(
			    "This is a tool that summarizes an English text. It takes an input `text` containing the text to summarize, "
			    "and returns a summary of the text."
			)
			SCREAMING_SNAKE_CASE     :   str									=				"summarizer"
			SCREAMING_SNAKE_CASE     :   str									=				AutoTokenizer
			SCREAMING_SNAKE_CASE     :   str									=				AutoModelForSeqaSeqLM

			SCREAMING_SNAKE_CASE     :   Optional[int]									=				["text"]
			SCREAMING_SNAKE_CASE     :   Optional[int]									=				["text"]







			def 				snake_case__(     self :				str							,   _UpperCamelCase :				int							)      ->Optional[int]:
									return self.pre_processor(_UpperCamelCase							,   return_tensors='''pt'''							,   truncation=_UpperCamelCase							)







			def 				snake_case__(     self :				Tuple							,   _UpperCamelCase :				Optional[int]							)      ->Tuple:
									return self.model.generate(**_UpperCamelCase							)[0]







			def 				snake_case__(     self :				Optional[Any]							,   _UpperCamelCase :				Optional[int]							)      ->Any:
									return self.pre_processor.decode(_UpperCamelCase							,   skip_special_tokens=_UpperCamelCase							,   clean_up_tokenization_spaces=_UpperCamelCase							)