

"""simple docstring"""






from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowerCAmelCase				      =   logging.get_logger(__name__)
class 	_lowerCAmelCase    (      __snake_case       ):
							__lowerCAmelCase					:    Union[str, Any]     =  ["input_ids", "attention_mask"]

							def __init__( self       :    Union[str, Any] ,		a       :    Optional[int]="[PAD]" ,		a       :    Union[str, Any]="[BOS]" ,		a       :    List[Any]="[EOS]" ,		a       :    List[Any]="[MASK]" ,		a       :    Optional[Any]="[CLS]" ,		a       :    Optional[int]="[SEP]" ,		a       :    Optional[Any]=2048 ,		**a       :    Optional[int] ,		)   ->    None:

									"""simple docstring"""
									lowercase										=							AddedToken(a ,		lstrip=a ,		rstrip=a    ) if isinstance(a ,		a    ) else pad_token
									lowercase										=							AddedToken(a ,		lstrip=a ,		rstrip=a    ) if isinstance(a ,		a    ) else bos_token
									lowercase										=							AddedToken(a ,		lstrip=a ,		rstrip=a    ) if isinstance(a ,		a    ) else eos_token
									lowercase										=							AddedToken(a ,		lstrip=a ,		rstrip=a    ) if isinstance(a ,		a    ) else mask_token
									lowercase										=							AddedToken(a ,		lstrip=a ,		rstrip=a    ) if isinstance(a ,		a    ) else cls_token
									lowercase										=							AddedToken(a ,		lstrip=a ,		rstrip=a    ) if isinstance(a ,		a    ) else sep_token

									super().__init__(
									    pad_token=a ,		bos_token=a ,		eos_token=a ,		mask_token=a ,		cls_token=a ,		sep_token=a ,		model_max_length=a ,		**a ,		)

									lowercase										=							2**8  # utf is 8 bits

									# define special tokens dict
									lowercase										=							{
									    self.pad_token: 0,
									    self.bos_token: 1,
									    self.eos_token: 2,
									    self.mask_token: 3,
									    self.cls_token: 4,
									    self.sep_token: 5,
									}
									lowercase										=							len(self.special_tokens_encoder    )
									lowercase										=							{v: k for k, v in self.special_tokens_encoder.items()}

							def 					_lowerCAmelCase  ( self       :    Dict    )   ->    Dict[str, int]:

									"""simple docstring"""
									lowercase										=							self.special_tokens_encoder.copy()
									vocab.update(self.added_tokens_encoder    )
									for i in range(self._utf_vocab_size    ):
											lowercase										=							chr(a    )
											lowercase										=							i + len(self.special_tokens_encoder    )
									return vocab

							@property
							def 					_lowerCAmelCase  ( self       :    str    )   ->    Tuple:

									"""simple docstring"""
									return self._utf_vocab_size + self._num_special_tokens

							def 					_lowerCAmelCase  ( self       :    Any ,		a       :    List[int] ,		a       :    Optional[List[int]] = None ,		a       :    bool = False    )   ->    List[int]:

									"""simple docstring"""







									if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=a ,		token_ids_a=a ,		already_has_special_tokens=a    )

									# normal case: some special tokens
									if token_ids_a is None:
											return [1] + [0] * len(a    ) + [1]
									return [1] + ([0] * len(a    )) + [1] + ([0] * len(a    )) + [1]

							def 					_lowerCAmelCase  ( self       :    int ,		a       :    List[int] ,		a       :    Optional[List[int]] = None    )   ->    List[int]:

									"""simple docstring"""







									if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
									else:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id] + token_ids_a + [self.sep_token_id]

							def 					_lowerCAmelCase  ( self       :    Dict ,		a       :    str    )   ->    List[str]:

									"""simple docstring"""
									lowercase										=							[chr(a    ) for i in text.encode('''utf-8'''    )]
									return tokens

							def 					_lowerCAmelCase  ( self       :    Optional[int] ,		a       :    Tuple    )   ->    int:

									"""simple docstring"""







									if token in self.special_tokens_encoder:
											lowercase										=							self.special_tokens_encoder[token]
									elif token in self.added_tokens_encoder:
											lowercase										=							self.added_tokens_encoder[token]
									elif len(a    ) != 1:
											lowercase										=							self.unk_token_id
									else:
											lowercase										=							ord(a    ) + self._num_special_tokens
									return token_id

							def 					_lowerCAmelCase  ( self       :    List[str] ,		a       :    int    )   ->    Dict:

									"""simple docstring"""







									if index in self.special_tokens_decoder:
											lowercase										=							self.special_tokens_decoder[index]
									elif index in self.added_tokens_decoder:
											lowercase										=							self.added_tokens_decoder[index]
									else:
											lowercase										=							chr(index - self._num_special_tokens    )
									return token

							def 					_lowerCAmelCase  ( self       :    Union[str, Any] ,		a       :    Dict    )   ->    Optional[Any]:

									"""simple docstring"""
									lowercase										=							B''''''
									for token in tokens:
											if token in self.special_tokens_decoder:
													lowercase										=							self.special_tokens_decoder[token].encode('''utf-8'''    )
											elif token in self.added_tokens_decoder:
													lowercase										=							self.special_tokens_decoder[token].encode('''utf-8'''    )
											elif token in self.special_tokens_encoder:
													lowercase										=							token.encode('''utf-8'''    )
											elif token in self.added_tokens_encoder:
													lowercase										=							token.encode('''utf-8'''    )
											else:
													lowercase										=							bytes([ord(a    )]    )
											bstring += tok_string
									lowercase										=							bstring.decode('''utf-8''' ,		errors='''replace'''    )
									return string

							def 					_lowerCAmelCase  ( self       :    Tuple ,		a       :    str ,		a       :    Optional[str] = None    )   ->    Tuple[str]:

									"""simple docstring"""
									return ()







'''simple docstring'''


import json
import os
import shutil
import tempfile
import unittest

from transformers import is_speech_available, is_torch_available
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_torch
from transformers.utils import FEATURE_EXTRACTOR_NAME


if is_speech_available() and is_torch_available():
					from transformers import SpeechTaFeatureExtractor, SpeechTaProcessor

					from .test_feature_extraction_speechta import floats_list


lowerCamelCase_					:			List[str]     						=				get_tests_dir('''fixtures/test_sentencepiece_bpe_char.model''')
@require_torch
class  _SCREAMING_SNAKE_CASE				(		unittest.TestCase						):


			'''simple docstring'''





			def 					A   ( self				:    Union[str, Any]      )				->     Union[str, Any]:

									'''simple docstring'''







									UpperCamelCase__       =			tempfile.mkdtemp()

									UpperCamelCase__       =			SpeechTaTokenizer(lowercase      )
									tokenizer.save_pretrained(self.tmpdirname      )

									UpperCamelCase__       =			{
									    """feature_size""": 1,
									    """padding_value""": 0.0,
									    """sampling_rate""": 1_6_0_0_0,
									    """do_normalize""": False,
									    """num_mel_bins""": 8_0,
									    """hop_length""": 1_6,
									    """win_length""": 6_4,
									    """win_function""": """hann_window""",
									    """fmin""": 8_0,
									    """fmax""": 7_6_0_0,
									    """mel_floor""": 1e-10,
									    """reduction_factor""": 2,
									    """return_attention_mask""": True,
									}

									UpperCamelCase__       =			os.path.join(self.tmpdirname			,   lowercase      )
									with open(self.feature_extraction_file			,   """w"""			,   encoding="""utf-8"""      ) as fp:
															fp.write(json.dumps(lowercase      ) + """\n"""      )






			def 					A   ( self				:    Union[str, Any]			,   **lowercase				:    List[Any]      )				->     Any:

									'''simple docstring'''







									return SpeechTaTokenizer.from_pretrained(self.tmpdirname			,   **lowercase      )






			def 					A   ( self				:    str			,   **lowercase				:    Tuple      )				->     Optional[Any]:

									'''simple docstring'''







									return SpeechTaFeatureExtractor.from_pretrained(self.tmpdirname			,   **lowercase      )






			def 					A   ( self				:    Union[str, Any]      )				->     str:

									'''simple docstring'''







									shutil.rmtree(self.tmpdirname      )






			def 					A   ( self				:    List[Any]      )				->     Optional[int]:

									'''simple docstring'''







									UpperCamelCase__       =			self.get_tokenizer()
									UpperCamelCase__       =			self.get_feature_extractor()

									UpperCamelCase__       =			SpeechTaProcessor(tokenizer=lowercase			,   feature_extractor=lowercase      )

									processor.save_pretrained(self.tmpdirname      )
									UpperCamelCase__       =			SpeechTaProcessor.from_pretrained(self.tmpdirname      )

									self.assertEqual(processor.tokenizer.get_vocab()			,   tokenizer.get_vocab()      )
									self.assertIsInstance(processor.tokenizer			,   lowercase      )

									self.assertEqual(processor.feature_extractor.to_json_string()			,   feature_extractor.to_json_string()      )
									self.assertIsInstance(processor.feature_extractor			,   lowercase      )






			def 					A   ( self				:    Tuple      )				->     Union[str, Any]:

									'''simple docstring'''







									UpperCamelCase__       =			SpeechTaProcessor(tokenizer=self.get_tokenizer()			,   feature_extractor=self.get_feature_extractor()      )
									processor.save_pretrained(self.tmpdirname      )

									UpperCamelCase__       =			self.get_tokenizer(bos_token="""(BOS)"""			,   eos_token="""(EOS)"""      )
									UpperCamelCase__       =			self.get_feature_extractor(do_normalize=lowercase			,   padding_value=1.0      )

									UpperCamelCase__       =			SpeechTaProcessor.from_pretrained(
									    self.tmpdirname			,   bos_token="""(BOS)"""			,   eos_token="""(EOS)"""			,   do_normalize=lowercase			,   padding_value=1.0      )

									self.assertEqual(processor.tokenizer.get_vocab()			,   tokenizer_add_kwargs.get_vocab()      )
									self.assertIsInstance(processor.tokenizer			,   lowercase      )

									self.assertEqual(processor.feature_extractor.to_json_string()			,   feature_extractor_add_kwargs.to_json_string()      )
									self.assertIsInstance(processor.feature_extractor			,   lowercase      )






			def 					A   ( self				:    List[Any]      )				->     Dict:

									'''simple docstring'''







									UpperCamelCase__       =			self.get_feature_extractor()
									UpperCamelCase__       =			self.get_tokenizer()

									UpperCamelCase__       =			SpeechTaProcessor(tokenizer=lowercase			,   feature_extractor=lowercase      )

									UpperCamelCase__       =			floats_list((3, 1_0_0_0)      )

									UpperCamelCase__       =			feature_extractor(audio=lowercase			,   return_tensors="""np"""      )
									UpperCamelCase__       =			processor(audio=lowercase			,   return_tensors="""np"""      )

									for key in input_feat_extract.keys():
															self.assertAlmostEqual(input_feat_extract[key].sum()			,   input_processor[key].sum()			,   delta=1e-2      )






			def 					A   ( self				:    List[str]      )				->     Optional[Any]:

									'''simple docstring'''







									UpperCamelCase__       =			self.get_feature_extractor()
									UpperCamelCase__       =			self.get_tokenizer()

									UpperCamelCase__       =			SpeechTaProcessor(tokenizer=lowercase			,   feature_extractor=lowercase      )

									UpperCamelCase__       =			floats_list((3, 1_0_0_0)      )

									UpperCamelCase__       =			feature_extractor(audio_target=lowercase			,   return_tensors="""np"""      )
									UpperCamelCase__       =			processor(audio_target=lowercase			,   return_tensors="""np"""      )

									for key in input_feat_extract.keys():
															self.assertAlmostEqual(input_feat_extract[key].sum()			,   input_processor[key].sum()			,   delta=1e-2      )






			def 					A   ( self				:    Optional[Any]      )				->     Optional[Any]:

									'''simple docstring'''







									UpperCamelCase__       =			self.get_feature_extractor()
									UpperCamelCase__       =			self.get_tokenizer()

									UpperCamelCase__       =			SpeechTaProcessor(tokenizer=lowercase			,   feature_extractor=lowercase      )

									UpperCamelCase__       =			"""This is a test string"""

									UpperCamelCase__       =			processor(text=lowercase      )
									UpperCamelCase__       =			tokenizer(lowercase      )

									for key in encoded_tok.keys():
															self.assertListEqual(encoded_tok[key]			,   encoded_processor[key]      )






			def 					A   ( self				:    List[Any]      )				->     int:

									'''simple docstring'''







									UpperCamelCase__       =			self.get_feature_extractor()
									UpperCamelCase__       =			self.get_tokenizer()

									UpperCamelCase__       =			SpeechTaProcessor(tokenizer=lowercase			,   feature_extractor=lowercase      )

									UpperCamelCase__       =			"""This is a test string"""

									UpperCamelCase__       =			processor(text_target=lowercase      )
									UpperCamelCase__       =			tokenizer(lowercase      )

									for key in encoded_tok.keys():
															self.assertListEqual(encoded_tok[key]			,   encoded_processor[key]      )






			def 					A   ( self				:    Any      )				->     Tuple:

									'''simple docstring'''







									UpperCamelCase__       =			self.get_feature_extractor()
									UpperCamelCase__       =			self.get_tokenizer()

									UpperCamelCase__       =			SpeechTaProcessor(tokenizer=lowercase			,   feature_extractor=lowercase      )

									UpperCamelCase__       =			[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

									UpperCamelCase__       =			processor.batch_decode(lowercase      )
									UpperCamelCase__       =			tokenizer.batch_decode(lowercase      )

									self.assertListEqual(lowercase			,   lowercase      )







			def 					A   ( self				:    List[Any]      )				->     List[str]:

									'''simple docstring'''







									UpperCamelCase__       =			self.get_feature_extractor()
									UpperCamelCase__       =			self.get_tokenizer()

									UpperCamelCase__       =			SpeechTaProcessor(tokenizer=lowercase			,   feature_extractor=lowercase      )

									self.assertListEqual(
									    processor.model_input_names			,   feature_extractor.model_input_names			,   msg="""`processor` and `feature_extractor` model input names do not match"""			,   )
