
'''simple docstring'''







import shutil
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SpeechaTextTokenizer, is_speech_available
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_torch, require_torchaudio
from transformers.utils import FEATURE_EXTRACTOR_NAME

from .test_feature_extraction_speech_to_text import floats_list


if is_speech_available():
							from transformers import SpeechaTextFeatureExtractor, SpeechaTextProcessor


_lowercase					:    Dict       				=			get_tests_dir("fixtures/test_sentencepiece.model")






@require_torch
@require_torchaudio
@require_sentencepiece
class 					lowerCAmelCase__  (    unittest.TestCase	):






					def       _snake_case       (  self						):


											"""simple docstring"""





											lowercase_      :	str						=			tempfile.mkdtemp()

											lowercase_      :	List[Any]						=			['''<s>''', '''<pad>''', '''</s>''', '''<unk>''', '''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']
											lowercase_      :	Tuple						=			dict(zip(__SCREAMING_SNAKE_CASE							,				range(len(__SCREAMING_SNAKE_CASE						)						)						)						)
											lowercase_      :	Tuple						=			Path(self.tmpdirname						)
											save_json(__SCREAMING_SNAKE_CASE							,				save_dir / VOCAB_FILES_NAMES['''vocab_file''']						)
											if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
																	copyfile(__SCREAMING_SNAKE_CASE							,				save_dir / VOCAB_FILES_NAMES['''spm_file''']						)

											lowercase_      :	Dict						=			SpeechaTextTokenizer.from_pretrained(self.tmpdirname						)
											tokenizer.save_pretrained(self.tmpdirname						)

											lowercase_      :	Union[str, Any]						=			{
											    '''feature_size''': 24,
											    '''num_mel_bins''': 24,
											    '''padding_value''': 0.0,
											    '''sampling_rate''': 1_60_00,
											    '''return_attention_mask''': False,
											    '''do_normalize''': True,
											}
											save_json(__SCREAMING_SNAKE_CASE							,				save_dir / FEATURE_EXTRACTOR_NAME						)






					def       _snake_case       (  self							,				**__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											return SpeechaTextTokenizer.from_pretrained(self.tmpdirname							,				**__SCREAMING_SNAKE_CASE						)






					def       _snake_case       (  self							,				**__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											return SpeechaTextFeatureExtractor.from_pretrained(self.tmpdirname							,				**__SCREAMING_SNAKE_CASE						)






					def       _snake_case       (  self						):


											"""simple docstring"""





											shutil.rmtree(self.tmpdirname						)






					def       _snake_case       (  self						):


											"""simple docstring"""





											lowercase_      :	List[str]						=			self.get_tokenizer()
											lowercase_      :	Dict						=			self.get_feature_extractor()

											lowercase_      :	Union[str, Any]						=			SpeechaTextProcessor(tokenizer=__SCREAMING_SNAKE_CASE							,				feature_extractor=__SCREAMING_SNAKE_CASE						)

											processor.save_pretrained(self.tmpdirname						)
											lowercase_      :	Any						=			SpeechaTextProcessor.from_pretrained(self.tmpdirname						)

											self.assertEqual(processor.tokenizer.get_vocab()							,				tokenizer.get_vocab()						)
											self.assertIsInstance(processor.tokenizer							,				__SCREAMING_SNAKE_CASE						)

											self.assertEqual(processor.feature_extractor.to_json_string()							,				feature_extractor.to_json_string()						)
											self.assertIsInstance(processor.feature_extractor							,				__SCREAMING_SNAKE_CASE						)






					def       _snake_case       (  self						):


											"""simple docstring"""





											lowercase_      :	str						=			SpeechaTextProcessor(
											    tokenizer=self.get_tokenizer()							,				feature_extractor=self.get_feature_extractor()						)
											processor.save_pretrained(self.tmpdirname						)

											lowercase_      :	Union[str, Any]						=			self.get_tokenizer(bos_token='''(BOS)'''							,				eos_token='''(EOS)'''						)
											lowercase_      :	List[str]						=			self.get_feature_extractor(do_normalize=__SCREAMING_SNAKE_CASE							,				padding_value=1.0						)

											lowercase_      :	int						=			SpeechaTextProcessor.from_pretrained(
											    self.tmpdirname							,				bos_token='''(BOS)'''							,				eos_token='''(EOS)'''							,				do_normalize=__SCREAMING_SNAKE_CASE							,				padding_value=1.0						)

											self.assertEqual(processor.tokenizer.get_vocab()							,				tokenizer_add_kwargs.get_vocab()						)
											self.assertIsInstance(processor.tokenizer							,				__SCREAMING_SNAKE_CASE						)

											self.assertEqual(processor.feature_extractor.to_json_string()							,				feature_extractor_add_kwargs.to_json_string()						)
											self.assertIsInstance(processor.feature_extractor							,				__SCREAMING_SNAKE_CASE						)






					def       _snake_case       (  self						):


											"""simple docstring"""





											lowercase_      :	List[Any]						=			self.get_feature_extractor()
											lowercase_      :	int						=			self.get_tokenizer()

											lowercase_      :	Optional[Any]						=			SpeechaTextProcessor(tokenizer=__SCREAMING_SNAKE_CASE							,				feature_extractor=__SCREAMING_SNAKE_CASE						)

											lowercase_      :	int						=			floats_list((3, 10_00)						)

											lowercase_      :	Any						=			feature_extractor(__SCREAMING_SNAKE_CASE							,				return_tensors='''np'''						)
											lowercase_      :	Dict						=			processor(__SCREAMING_SNAKE_CASE							,				return_tensors='''np'''						)

											for key in input_feat_extract.keys():
																	self.assertAlmostEqual(input_feat_extract[key].sum()							,				input_processor[key].sum()							,				delta=1E-2						)






					def       _snake_case       (  self						):


											"""simple docstring"""





											lowercase_      :	int						=			self.get_feature_extractor()
											lowercase_      :	Optional[int]						=			self.get_tokenizer()

											lowercase_      :	List[Any]						=			SpeechaTextProcessor(tokenizer=__SCREAMING_SNAKE_CASE							,				feature_extractor=__SCREAMING_SNAKE_CASE						)

											lowercase_      :	List[Any]						=			'''This is a test string'''

											lowercase_      :	str						=			processor(text=__SCREAMING_SNAKE_CASE						)

											lowercase_      :	str						=			tokenizer(__SCREAMING_SNAKE_CASE						)

											for key in encoded_tok.keys():
																	self.assertListEqual(encoded_tok[key]							,				encoded_processor[key]						)






					def       _snake_case       (  self						):


											"""simple docstring"""





											lowercase_      :	str						=			self.get_feature_extractor()
											lowercase_      :	Union[str, Any]						=			self.get_tokenizer()

											lowercase_      :	Optional[Any]						=			SpeechaTextProcessor(tokenizer=__SCREAMING_SNAKE_CASE							,				feature_extractor=__SCREAMING_SNAKE_CASE						)

											lowercase_      :	Any						=			[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

											lowercase_      :	List[str]						=			processor.batch_decode(__SCREAMING_SNAKE_CASE						)
											lowercase_      :	List[str]						=			tokenizer.batch_decode(__SCREAMING_SNAKE_CASE						)

											self.assertListEqual(__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE						)






					def       _snake_case       (  self						):


											"""simple docstring"""





											lowercase_      :	str						=			self.get_feature_extractor()
											lowercase_      :	Tuple						=			self.get_tokenizer()

											lowercase_      :	Union[str, Any]						=			SpeechaTextProcessor(tokenizer=__SCREAMING_SNAKE_CASE							,				feature_extractor=__SCREAMING_SNAKE_CASE						)

											self.assertListEqual(
											    processor.model_input_names							,				feature_extractor.model_input_names							,				msg='''`processor` and `feature_extractor` model input names do not match'''							,				)





'''simple docstring'''







import posixpath
from pathlib import Path
from unittest.mock import patch

import pytest
from fsspec.implementations.local import AbstractFileSystem, LocalFileSystem, stringify_path
from fsspec.registry import _registry as _fsspec_registry






class 					lowerCAmelCase__  (    lowerCamelCase_	):
					lowerCAmelCase_	  =	'''mock'''






					def __init__(  self							,				*__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											super().__init__()
											lowercase_      :	int						=			LocalFileSystem(*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						)
											lowercase_      :	Tuple						=			Path(__SCREAMING_SNAKE_CASE						).resolve().as_posix() + '''/'''






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE							,				*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											lowercase_      :	List[str]						=			posixpath.join(self.local_root_dir							,				self._strip_protocol(__SCREAMING_SNAKE_CASE						)						)
											return self._fs.mkdir(__SCREAMING_SNAKE_CASE							,				*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						)






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE							,				*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											lowercase_      :	Optional[int]						=			posixpath.join(self.local_root_dir							,				self._strip_protocol(__SCREAMING_SNAKE_CASE						)						)
											return self._fs.makedirs(__SCREAMING_SNAKE_CASE							,				*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						)






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											lowercase_      :	Any						=			posixpath.join(self.local_root_dir							,				self._strip_protocol(__SCREAMING_SNAKE_CASE						)						)
											return self._fs.rmdir(__SCREAMING_SNAKE_CASE						)






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE=True							,				*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											lowercase_      :	str						=			posixpath.join(self.local_root_dir							,				self._strip_protocol(__SCREAMING_SNAKE_CASE						)						)
											lowercase_      :	Union[str, Any]						=			self._fs.ls(__SCREAMING_SNAKE_CASE							,				detail=__SCREAMING_SNAKE_CASE							,				*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						)
											if detail:
																	return [{**info, "name": info["name"][len(self.local_root_dir						) :]} for info in out]
											else:
																	return [name[len(self.local_root_dir						) :] for name in out]






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE							,				*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											lowercase_      :	Optional[int]						=			posixpath.join(self.local_root_dir							,				self._strip_protocol(__SCREAMING_SNAKE_CASE						)						)
											lowercase_      :	Optional[Any]						=			dict(self._fs.info(__SCREAMING_SNAKE_CASE							,				*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						)						)
											lowercase_      :	Dict						=			out['''name'''][len(self.local_root_dir						) :]
											return out






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE							,				*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											lowercase_      :	List[str]						=			posixpath.join(self.local_root_dir							,				self._strip_protocol(__SCREAMING_SNAKE_CASE						)						)
											lowercase_      :	str						=			posixpath.join(self.local_root_dir							,				self._strip_protocol(__SCREAMING_SNAKE_CASE						)						)
											return self._fs.cp_file(__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE							,				*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						)






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE							,				*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											lowercase_      :	List[Any]						=			posixpath.join(self.local_root_dir							,				self._strip_protocol(__SCREAMING_SNAKE_CASE						)						)
											return self._fs.rm_file(__SCREAMING_SNAKE_CASE							,				*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						)






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE							,				*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											lowercase_      :	Union[str, Any]						=			posixpath.join(self.local_root_dir							,				self._strip_protocol(__SCREAMING_SNAKE_CASE						)						)
											return self._fs.rm(__SCREAMING_SNAKE_CASE							,				*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						)






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE							,				*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											lowercase_      :	Union[str, Any]						=			posixpath.join(self.local_root_dir							,				self._strip_protocol(__SCREAMING_SNAKE_CASE						)						)
											return self._fs._open(__SCREAMING_SNAKE_CASE							,				*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						)






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											lowercase_      :	Dict						=			posixpath.join(self.local_root_dir							,				self._strip_protocol(__SCREAMING_SNAKE_CASE						)						)
											return self._fs.created(__SCREAMING_SNAKE_CASE						)






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											lowercase_      :	Optional[Any]						=			posixpath.join(self.local_root_dir							,				self._strip_protocol(__SCREAMING_SNAKE_CASE						)						)
											return self._fs.modified(__SCREAMING_SNAKE_CASE						)






					@classmethod
					def       _snake_case       (  cls							,				__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											lowercase_      :	Optional[Any]						=			stringify_path(__SCREAMING_SNAKE_CASE						)
											if path.startswith('''mock://'''						):
																	lowercase_      :	int						=			path[7:]
											return path







class 					lowerCAmelCase__  (    lowerCamelCase_	):
					lowerCAmelCase_	  =	'''tmp'''
					lowerCAmelCase_	  =	None






					def __init__(  self							,				*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											assert self.tmp_dir is not None, "TmpDirFileSystem.tmp_dir is not set"
											super().__init__(*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE							,				local_root_dir=self.tmp_dir							,				auto_mkdir=__SCREAMING_SNAKE_CASE						)






					@classmethod
					def       _snake_case       (  cls							,				__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											lowercase_      :	str						=			stringify_path(__SCREAMING_SNAKE_CASE						)
											if path.startswith('''tmp://'''						):
																	lowercase_      :	Union[str, Any]						=			path[6:]
											return path







@pytest.fixture
def    snake_case_						(							):



						"""simple docstring"""





						lowercase_      :	List[Any]						=			MockFileSystem
						lowercase_      :	Tuple						=			TmpDirFileSystem
						yield
						del _fsspec_registry["mock"]
						del _fsspec_registry["tmp"]
@pytest.fixture
def    snake_case_						(							__SCREAMING_SNAKE_CASE   :					List[Any] , __SCREAMING_SNAKE_CASE   :					List[Any]				):



						"""simple docstring"""





						lowercase_      :	List[str]						=			tmp_path_factory.mktemp('''mockfs'''				)
						return MockFileSystem(local_root_dir=__SCREAMING_SNAKE_CASE , auto_mkdir=__SCREAMING_SNAKE_CASE				)




@pytest.fixture
def    snake_case_						(							__SCREAMING_SNAKE_CASE   :					Union[str, Any] , __SCREAMING_SNAKE_CASE   :					Any				):



						"""simple docstring"""





						lowercase_      :	Union[str, Any]						=			tmp_path_factory.mktemp('''tmpfs'''				)
						with patch.object(__SCREAMING_SNAKE_CASE , '''tmp_dir''' , __SCREAMING_SNAKE_CASE				):
												yield TmpDirFileSystem()



