




'''simple docstring'''




import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


a_      :		str							      =       get_tests_dir("""fixtures/test_sentencepiece.model""")

if is_sentencepiece_available():
     import sentencepiece as sp


a_      :		Optional[Any]							      =       5
a_      :		str							      =       10


@require_sentencepiece
@require_tokenizers
class   __UpperCamelCase       ( lowerCamelCase__			,					unittest.TestCase				):
       lowercase	:    int      						=SpeechaTextTokenizer
       lowercase	:    int      						=False
       lowercase	:    List[str]      						=True






       def      lowercase__     (							self  ):



              """simple docstring"""







              super().setUp()

              lowerCamelCase_							=sp.SentencePieceProcessor()
              spm_model.Load(lowerCAmelCase  )
              lowerCamelCase_							=['''<s>''', '''<pad>''', '''</s>''', '''<unk>''']

              vocab += [spm_model.IdToPiece(id_  ) for id_ in range(len(lowerCAmelCase  )  )]
              lowerCamelCase_							=dict(zip(lowerCAmelCase,		range(len(lowerCAmelCase  )  )  )  )

              lowerCamelCase_							=Path(self.tmpdirname  )
              save_json(lowerCAmelCase,		save_dir / VOCAB_FILES_NAMES['''vocab_file''']  )
              if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
                     copyfile(lowerCAmelCase,		save_dir / VOCAB_FILES_NAMES['''spm_file''']  )

              lowerCamelCase_							=SpeechaTextTokenizer.from_pretrained(self.tmpdirname  )
              tokenizer.save_pretrained(self.tmpdirname  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							='''<pad>'''
              lowerCamelCase_							=1

              self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase  ),		lowerCAmelCase  )
              self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase  ),		lowerCAmelCase  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=list(self.get_tokenizer().get_vocab().keys()  )

              self.assertEqual(vocab_keys[0],		'''<s>'''  )
              self.assertEqual(vocab_keys[1],		'''<pad>'''  )
              self.assertEqual(vocab_keys[-1],		'''j'''  )
              self.assertEqual(len(lowerCAmelCase  ),		1_001  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              self.assertEqual(self.get_tokenizer().vocab_size,		1_001  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=SpeechaTextTokenizer.from_pretrained(self.tmpdirname  )

              lowerCamelCase_							=tokenizer.tokenize('''This is a test'''  )
              self.assertListEqual(lowerCAmelCase,		['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']  )

              self.assertListEqual(
                  tokenizer.convert_tokens_to_ids(lowerCAmelCase  ),		[289, 50, 14, 174, 386],		)

              lowerCamelCase_							=tokenizer.tokenize('''I was born in 92000, and this is falsé.'''  )
              self.assertListEqual(
                  lowerCAmelCase,		[SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''9''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''é''', '''.'''],		)
              lowerCamelCase_							=tokenizer.convert_tokens_to_ids(lowerCAmelCase  )
              self.assertListEqual(lowerCAmelCase,		[12, 25, 88, 59, 28, 23, 11, 4, 606, 351, 351, 351, 7, 16, 70, 50, 76, 84, 10, 4, 8]  )

              lowerCamelCase_							=tokenizer.convert_ids_to_tokens(lowerCAmelCase  )
              self.assertListEqual(
                  lowerCAmelCase,		[SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''<unk>''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''<unk>''', '''.'''],		)






       @slow
       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							={'''input_ids''': [[3_791, 797, 31, 11, 64, 797, 31, 2_429, 433, 12, 1_176, 12, 20, 786, 915, 142, 2_413, 240, 37, 3_238, 797, 31, 11, 35, 93, 915, 142, 2_413, 240, 37, 5_540, 567, 1_276, 93, 37, 610, 40, 62, 455, 657, 1_042, 123, 780, 177, 37, 309, 241, 1_298, 514, 20, 292, 2_737, 114, 2_469, 241, 85, 64, 302, 548, 528, 423, 4, 509, 406, 423, 37, 601, 4, 777, 302, 548, 528, 423, 284, 4, 3_388, 511, 459, 4, 3_555, 40, 321, 302, 705, 4, 3_388, 511, 583, 326, 5, 5, 5, 62, 3_310, 560, 177, 2_680, 217, 1_508, 32, 31, 853, 418, 64, 583, 511, 1_605, 62, 35, 93, 560, 177, 2_680, 217, 1_508, 1_521, 64, 583, 511, 519, 62, 20, 1_515, 764, 20, 149, 261, 5_625, 7_972, 20, 5_540, 567, 1_276, 93, 3_925, 1_675, 11, 15, 802, 7_972, 576, 217, 1_508, 11, 35, 93, 1_253, 2_441, 15, 289, 652, 31, 416, 321, 3_842, 115, 40, 911, 8, 476, 619, 4, 380, 142, 423, 335, 240, 35, 93, 264, 8, 11, 335, 569, 420, 163, 5, 2], [260, 548, 528, 423, 20, 451, 20, 2_681, 1_153, 3_434, 20, 5_540, 37, 567, 126, 1_253, 2_441, 3_376, 449, 210, 431, 1_563, 177, 767, 5_540, 11, 1_203, 472, 11, 2_953, 685, 285, 364, 706, 1_153, 20, 6_799, 20, 2_869, 20, 4_464, 126, 40, 2_429, 20, 1_040, 866, 2_664, 418, 20, 318, 20, 1_726, 186, 20, 265, 522, 35, 93, 2_191, 4_634, 20, 1_040, 12, 6_799, 15, 228, 2_356, 142, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2_575, 2_666, 684, 1_582, 1_176, 12, 627, 149, 619, 20, 4_902, 563, 11, 20, 149, 261, 3_420, 2_356, 174, 142, 4_714, 131, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
              # fmt: on

              self.tokenizer_integration_test_util(
                  expected_encoding=lowerCAmelCase,		model_name='''facebook/s2t-small-mustc-en-de-st''',		revision='''a14f04cf0776c02f62a8cb800cf7909e15ea23ad''',		)





@require_sentencepiece
class   __UpperCamelCase       ( unittest.TestCase				):
       lowercase	:    Tuple      						='valhalla/s2t_mustc_multilinguial_medium'

       lowercase	:    Dict      						='C\'est trop cool'
       lowercase	:    str      						='Esto es genial'






       @classmethod
       def      lowercase__     (							cls  ):



              """simple docstring"""







              lowerCamelCase_							=SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name  )
              return cls






       def      lowercase__     (							self  ):



              """simple docstring"""







              self.assertEqual(self.tokenizer.lang_code_to_id['''pt'''],		4  )
              self.assertEqual(self.tokenizer.lang_code_to_id['''ru'''],		6  )
              self.assertEqual(self.tokenizer.lang_code_to_id['''it'''],		9  )
              self.assertEqual(self.tokenizer.lang_code_to_id['''de'''],		11  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              self.assertEqual(self.tokenizer.vocab_size,		10_000  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              self.assertIn(lowerCAmelCase,		self.tokenizer.all_special_ids  )
              lowerCamelCase_							=[ES_CODE, 4, 1_601, 47, 7_647, 2]
              lowerCamelCase_							=self.tokenizer.decode(lowerCAmelCase,		skip_special_tokens=lowerCAmelCase  )
              lowerCamelCase_							=self.tokenizer.decode(generated_ids[1:],		skip_special_tokens=lowerCAmelCase  )
              self.assertEqual(lowerCAmelCase,		lowerCAmelCase  )
              self.assertNotIn(self.tokenizer.eos_token,		lowerCAmelCase  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							='''fr'''
              lowerCamelCase_							=self.tokenizer(self.french_text  ).input_ids
              self.assertEqual(encoded[0],		lowerCAmelCase  )
              self.assertEqual(encoded[-1],		self.tokenizer.eos_token_id  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							='''fr'''
              self.assertListEqual(self.tokenizer.prefix_tokens,		[FR_CODE]  )

              lowerCamelCase_							='''es'''
              self.assertListEqual(self.tokenizer.prefix_tokens,		[ES_CODE]  )





import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.text import TextDatasetReader

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases




def       _snake_case	(   lowerCAmelCase							:   str   , lowerCAmelCase							:   Union[str, Any]	):
		"""simple docstring"""







		assert isinstance(lowerCAmelCase   , lowerCAmelCase	)
		assert dataset.num_rows == 4
		assert dataset.num_columns == 1
		assert dataset.column_names == ["text"]
		for feature, expected_dtype in expected_features.items():
				assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize("keep_in_memory"   , [False, True]	)
def       _snake_case	(   lowerCAmelCase							:   Tuple   , lowerCAmelCase							:   Union[str, Any]   , lowerCAmelCase							:   int	):
		"""simple docstring"""







		SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    tmp_path / "cache"
		SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    {"text": "string"}
		with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
				SCREAMING_SNAKE_CASE_  :		List[str]				      =    TextDatasetReader(lowerCAmelCase   , cache_dir=lowerCAmelCase   , keep_in_memory=lowerCAmelCase	).read()
		_check_text_dataset(lowerCAmelCase   , lowerCAmelCase	)




@pytest.mark.parametrize(
    "features"   , [
        None,
        {"text": "string"},
        {"text": "int32"},
        {"text": "float32"},
    ]   , )
def       _snake_case	(   lowerCAmelCase							:   Tuple   , lowerCAmelCase							:   Dict   , lowerCAmelCase							:   List[str]	):
		"""simple docstring"""







		SCREAMING_SNAKE_CASE_  :		Any				      =    tmp_path / "cache"
		SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    {"text": "string"}
		SCREAMING_SNAKE_CASE_  :		Any				      =    features.copy() if features else default_expected_features
		SCREAMING_SNAKE_CASE_  :		List[str]				      =    (
		    Features({feature: Value(lowerCAmelCase	) for feature, dtype in features.items()}	) if features is not None else None
		)
		SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    TextDatasetReader(lowerCAmelCase   , features=lowerCAmelCase   , cache_dir=lowerCAmelCase	).read()
		_check_text_dataset(lowerCAmelCase   , lowerCAmelCase	)




@pytest.mark.parametrize("split"   , [None, NamedSplit("train"	), "train", "test"]	)
def       _snake_case	(   lowerCAmelCase							:   Union[str, Any]   , lowerCAmelCase							:   Optional[Any]   , lowerCAmelCase							:   List[str]	):
		"""simple docstring"""







		SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    tmp_path / "cache"
		SCREAMING_SNAKE_CASE_  :		List[str]				      =    {"text": "string"}
		SCREAMING_SNAKE_CASE_  :		List[str]				      =    TextDatasetReader(lowerCAmelCase   , cache_dir=lowerCAmelCase   , split=lowerCAmelCase	).read()
		_check_text_dataset(lowerCAmelCase   , lowerCAmelCase	)
		assert dataset.split == split if split else "train"




@pytest.mark.parametrize("path_type"   , [str, list]	)
def       _snake_case	(   lowerCAmelCase							:   int   , lowerCAmelCase							:   int   , lowerCAmelCase							:   Optional[int]	):
		"""simple docstring"""







		if issubclass(lowerCAmelCase   , lowerCAmelCase	):
				SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    text_path
		elif issubclass(lowerCAmelCase   , lowerCAmelCase	):
				SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    [text_path]
		SCREAMING_SNAKE_CASE_  :		int				      =    tmp_path / "cache"
		SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    {"text": "string"}
		SCREAMING_SNAKE_CASE_  :		List[str]				      =    TextDatasetReader(lowerCAmelCase   , cache_dir=lowerCAmelCase	).read()
		_check_text_dataset(lowerCAmelCase   , lowerCAmelCase	)




def       _snake_case	(   lowerCAmelCase							:   Dict   , lowerCAmelCase							:   Dict   , lowerCAmelCase							:   List[str]=("train",)	):
		"""simple docstring"""







		assert isinstance(lowerCAmelCase   , lowerCAmelCase	)
		for split in splits:
				SCREAMING_SNAKE_CASE_  :		int				      =    dataset_dict[split]
				assert dataset.num_rows == 4
				assert dataset.num_columns == 1
				assert dataset.column_names == ["text"]
				for feature, expected_dtype in expected_features.items():
						assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize("keep_in_memory"   , [False, True]	)
def       _snake_case	(   lowerCAmelCase							:   List[str]   , lowerCAmelCase							:   str   , lowerCAmelCase							:   Dict	):
		"""simple docstring"""







		SCREAMING_SNAKE_CASE_  :		List[str]				      =    tmp_path / "cache"
		SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    {"text": "string"}
		with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
				SCREAMING_SNAKE_CASE_  :		List[Any]				      =    TextDatasetReader({"train": text_path}   , cache_dir=lowerCAmelCase   , keep_in_memory=lowerCAmelCase	).read()
		_check_text_datasetdict(lowerCAmelCase   , lowerCAmelCase	)




@pytest.mark.parametrize(
    "features"   , [
        None,
        {"text": "string"},
        {"text": "int32"},
        {"text": "float32"},
    ]   , )
def       _snake_case	(   lowerCAmelCase							:   Union[str, Any]   , lowerCAmelCase							:   List[Any]   , lowerCAmelCase							:   Tuple	):
		"""simple docstring"""







		SCREAMING_SNAKE_CASE_  :		List[Any]				      =    tmp_path / "cache"
		# CSV file loses col_1 string dtype information: default now is "int64" instead of "string"
		SCREAMING_SNAKE_CASE_  :		Tuple				      =    {"text": "string"}
		SCREAMING_SNAKE_CASE_  :		Any				      =    features.copy() if features else default_expected_features
		SCREAMING_SNAKE_CASE_  :		Dict				      =    (
		    Features({feature: Value(lowerCAmelCase	) for feature, dtype in features.items()}	) if features is not None else None
		)
		SCREAMING_SNAKE_CASE_  :		str				      =    TextDatasetReader({"train": text_path}   , features=lowerCAmelCase   , cache_dir=lowerCAmelCase	).read()
		_check_text_datasetdict(lowerCAmelCase   , lowerCAmelCase	)







@pytest.mark.parametrize("split"   , [None, NamedSplit("train"	), "train", "test"]	)
def       _snake_case	(   lowerCAmelCase							:   Optional[Any]   , lowerCAmelCase							:   Any   , lowerCAmelCase							:   Dict	):
		"""simple docstring"""







		if split:
				SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    {split: text_path}
		else:
				SCREAMING_SNAKE_CASE_  :		List[Any]				      =    "train"
				SCREAMING_SNAKE_CASE_  :		Tuple				      =    {"train": text_path, "test": text_path}
		SCREAMING_SNAKE_CASE_  :		Any				      =    tmp_path / "cache"
		SCREAMING_SNAKE_CASE_  :		List[str]				      =    {"text": "string"}
		SCREAMING_SNAKE_CASE_  :		str				      =    TextDatasetReader(lowerCAmelCase   , cache_dir=lowerCAmelCase	).read()
		_check_text_datasetdict(lowerCAmelCase   , lowerCAmelCase   , splits=list(path.keys()	)	)
		assert all(dataset[split].split == split for split in path.keys()	)
