


"""simple docstring"""





import json
import os
import shutil
import tempfile
import unittest

from transformers import is_speech_available, is_torch_available
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_torch
from transformers.utils import FEATURE_EXTRACTOR_NAME


if is_speech_available() and is_torch_available():
 from transformers import SpeechTaFeatureExtractor, SpeechTaProcessor

 from .test_feature_extraction_speechta import floats_list


snake_case__		:  Tuple     					=					get_tests_dir('''fixtures/test_sentencepiece_bpe_char.model''')



@require_torch
class 		snake_case_(							unittest.TestCase      ):






     def   lowerCamelCase__  (						self	:       int					):
           lowerCAmelCase       :						List[Any]							= tempfile.mkdtemp()

           lowerCAmelCase       :						Tuple							= SpeechTaTokenizer(__lowercase					)
           tokenizer.save_pretrained(self.tmpdirname					)

           lowerCAmelCase       :						Tuple							= {
               'feature_size': 1,
               'padding_value': 0.0,
               'sampling_rate': 1_6_0_0_0,
               'do_normalize': False,
               'num_mel_bins': 8_0,
               'hop_length': 1_6,
               'win_length': 6_4,
               'win_function': 'hann_window',
               'fmin': 8_0,
               'fmax': 7_6_0_0,
               'mel_floor': 1E-10,
               'reduction_factor': 2,
               'return_attention_mask': True,
           }

           lowerCAmelCase       :						str							= os.path.join(self.tmpdirname       ,					__lowercase					)
           with open(self.feature_extraction_file       ,					'''w'''       ,					encoding='''utf-8'''					) as fp:
                 fp.write(json.dumps(__lowercase					) + '''\n'''					)






     def   lowerCamelCase__  (						self	:       Optional[Any]       ,					**UpperCamelCase_	:       int					):
           return SpeechTaTokenizer.from_pretrained(self.tmpdirname       ,					**__lowercase					)






     def   lowerCamelCase__  (						self	:       int       ,					**UpperCamelCase_	:       List[Any]					):
           return SpeechTaFeatureExtractor.from_pretrained(self.tmpdirname       ,					**__lowercase					)






     def   lowerCamelCase__  (						self	:       Dict					):
           shutil.rmtree(self.tmpdirname					)






     def   lowerCamelCase__  (						self	:       Dict					):
           lowerCAmelCase       :						Optional[Any]							= self.get_tokenizer()
           lowerCAmelCase       :						Tuple							= self.get_feature_extractor()

           lowerCAmelCase       :						Dict							= SpeechTaProcessor(tokenizer=__lowercase       ,					feature_extractor=__lowercase					)

           processor.save_pretrained(self.tmpdirname					)
           lowerCAmelCase       :						int							= SpeechTaProcessor.from_pretrained(self.tmpdirname					)

           self.assertEqual(processor.tokenizer.get_vocab()       ,					tokenizer.get_vocab()					)
           self.assertIsInstance(processor.tokenizer       ,					__lowercase					)

           self.assertEqual(processor.feature_extractor.to_json_string()       ,					feature_extractor.to_json_string()					)
           self.assertIsInstance(processor.feature_extractor       ,					__lowercase					)






     def   lowerCamelCase__  (						self	:       Union[str, Any]					):
           lowerCAmelCase       :						str							= SpeechTaProcessor(tokenizer=self.get_tokenizer()       ,					feature_extractor=self.get_feature_extractor()					)
           processor.save_pretrained(self.tmpdirname					)

           lowerCAmelCase       :						Any							= self.get_tokenizer(bos_token='''(BOS)'''       ,					eos_token='''(EOS)'''					)
           lowerCAmelCase       :						Optional[Any]							= self.get_feature_extractor(do_normalize=__lowercase       ,					padding_value=1.0					)

           lowerCAmelCase       :						Tuple							= SpeechTaProcessor.from_pretrained(
               self.tmpdirname       ,					bos_token='''(BOS)'''       ,					eos_token='''(EOS)'''       ,					do_normalize=__lowercase       ,					padding_value=1.0					)

           self.assertEqual(processor.tokenizer.get_vocab()       ,					tokenizer_add_kwargs.get_vocab()					)
           self.assertIsInstance(processor.tokenizer       ,					__lowercase					)

           self.assertEqual(processor.feature_extractor.to_json_string()       ,					feature_extractor_add_kwargs.to_json_string()					)
           self.assertIsInstance(processor.feature_extractor       ,					__lowercase					)






     def   lowerCamelCase__  (						self	:       Optional[Any]					):
           lowerCAmelCase       :						List[str]							= self.get_feature_extractor()
           lowerCAmelCase       :						Optional[Any]							= self.get_tokenizer()

           lowerCAmelCase       :						Any							= SpeechTaProcessor(tokenizer=__lowercase       ,					feature_extractor=__lowercase					)

           lowerCAmelCase       :						Dict							= floats_list((3, 1_0_0_0)					)

           lowerCAmelCase       :						Optional[Any]							= feature_extractor(audio=__lowercase       ,					return_tensors='''np'''					)
           lowerCAmelCase       :						Optional[int]							= processor(audio=__lowercase       ,					return_tensors='''np'''					)

           for key in input_feat_extract.keys():
                 self.assertAlmostEqual(input_feat_extract[key].sum()       ,					input_processor[key].sum()       ,					delta=1E-2					)






     def   lowerCamelCase__  (						self	:       Tuple					):
           lowerCAmelCase       :						List[Any]							= self.get_feature_extractor()
           lowerCAmelCase       :						List[str]							= self.get_tokenizer()

           lowerCAmelCase       :						Any							= SpeechTaProcessor(tokenizer=__lowercase       ,					feature_extractor=__lowercase					)

           lowerCAmelCase       :						str							= floats_list((3, 1_0_0_0)					)

           lowerCAmelCase       :						str							= feature_extractor(audio_target=__lowercase       ,					return_tensors='''np'''					)
           lowerCAmelCase       :						Union[str, Any]							= processor(audio_target=__lowercase       ,					return_tensors='''np'''					)

           for key in input_feat_extract.keys():
                 self.assertAlmostEqual(input_feat_extract[key].sum()       ,					input_processor[key].sum()       ,					delta=1E-2					)






     def   lowerCamelCase__  (						self	:       Dict					):
           lowerCAmelCase       :						Optional[int]							= self.get_feature_extractor()
           lowerCAmelCase       :						int							= self.get_tokenizer()

           lowerCAmelCase       :						Union[str, Any]							= SpeechTaProcessor(tokenizer=__lowercase       ,					feature_extractor=__lowercase					)

           lowerCAmelCase       :						Optional[int]							= 'This is a test string'

           lowerCAmelCase       :						List[Any]							= processor(text=__lowercase					)
           lowerCAmelCase       :						int							= tokenizer(__lowercase					)

           for key in encoded_tok.keys():
                 self.assertListEqual(encoded_tok[key]       ,					encoded_processor[key]					)






     def   lowerCamelCase__  (						self	:       Dict					):
           lowerCAmelCase       :						Optional[int]							= self.get_feature_extractor()
           lowerCAmelCase       :						Optional[Any]							= self.get_tokenizer()

           lowerCAmelCase       :						Optional[Any]							= SpeechTaProcessor(tokenizer=__lowercase       ,					feature_extractor=__lowercase					)

           lowerCAmelCase       :						str							= 'This is a test string'

           lowerCAmelCase       :						List[Any]							= processor(text_target=__lowercase					)
           lowerCAmelCase       :						int							= tokenizer(__lowercase					)

           for key in encoded_tok.keys():
                 self.assertListEqual(encoded_tok[key]       ,					encoded_processor[key]					)






     def   lowerCamelCase__  (						self	:       Optional[Any]					):
           lowerCAmelCase       :						Dict							= self.get_feature_extractor()
           lowerCAmelCase       :						Any							= self.get_tokenizer()

           lowerCAmelCase       :						Any							= SpeechTaProcessor(tokenizer=__lowercase       ,					feature_extractor=__lowercase					)

           lowerCAmelCase       :						Union[str, Any]							= [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

           lowerCAmelCase       :						Dict							= processor.batch_decode(__lowercase					)
           lowerCAmelCase       :						List[Any]							= tokenizer.batch_decode(__lowercase					)

           self.assertListEqual(__lowercase       ,					__lowercase					)






     def   lowerCamelCase__  (						self	:       List[Any]					):
           lowerCAmelCase       :						List[Any]							= self.get_feature_extractor()
           lowerCAmelCase       :						str							= self.get_tokenizer()

           lowerCAmelCase       :						List[Any]							= SpeechTaProcessor(tokenizer=__lowercase       ,					feature_extractor=__lowercase					)

           self.assertListEqual(
               processor.model_input_names       ,					feature_extractor.model_input_names       ,					msg='''`processor` and `feature_extractor` model input names do not match'''       ,					)








import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase__						=		logging.get_logger(__name__)

UpperCamelCase__						=		{}


class 							__SCREAMING_SNAKE_CASE				(						lowerCamelCase    ):
					_lowerCamelCase						    =					"""fsmt_decoder"""

					def __init__(							self					,			__lowercase=0					,			__lowercase=0   ):







						'''simple docstring'''




						super().__init__()
						lowerCamelCase_				:						Tuple				  =				vocab_size
						lowerCamelCase_				:						Union[str, Any]				  =				bos_token_id


class 							__SCREAMING_SNAKE_CASE				(						lowerCamelCase    ):
					_lowerCamelCase						    =					"""fsmt"""
					_lowerCamelCase						    =					{"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}

					def __init__(							self					,			__lowercase=["en", "de"]					,			__lowercase=42_024					,			__lowercase=42_024					,			__lowercase="relu"					,			__lowercase=1_024					,			__lowercase=200					,			__lowercase=1_024					,			__lowercase=4_096					,			__lowercase=12					,			__lowercase=16					,			__lowercase=0.0					,			__lowercase=4_096					,			__lowercase=12					,			__lowercase=16					,			__lowercase=0.0					,			__lowercase=0.0					,			__lowercase=0.1					,			__lowercase=0.0					,			__lowercase=0.02					,			__lowercase=2					,			__lowercase=True					,			__lowercase=True					,			__lowercase=False					,			__lowercase=5					,			__lowercase=1.0					,			__lowercase=False					,			__lowercase=True					,			__lowercase=1					,			__lowercase=0					,			__lowercase=2					,			__lowercase=2					,			**__lowercase					,			):







						'''simple docstring'''




						lowerCamelCase_				:						Any				  =				langs
						lowerCamelCase_				:						Union[str, Any]				  =				src_vocab_size
						lowerCamelCase_				:						Tuple				  =				tgt_vocab_size
						lowerCamelCase_				:						List[Any]				  =				d_model  # encoder_embed_dim and decoder_embed_dim

						lowerCamelCase_				:						str				  =				encoder_ffn_dim
						lowerCamelCase_				:						str				  =				encoder_layers
						lowerCamelCase_				:						Dict				  =				encoder_attention_heads
						lowerCamelCase_				:						Any				  =				encoder_layerdrop
						lowerCamelCase_				:						Union[str, Any]				  =				decoder_layerdrop
						lowerCamelCase_				:						Optional[int]				  =				decoder_ffn_dim
						lowerCamelCase_				:						List[str]				  =				decoder_layers
						lowerCamelCase_				:						int				  =				decoder_attention_heads
						lowerCamelCase_				:						List[str]				  =				max_position_embeddings
						lowerCamelCase_				:						int				  =				init_std  # Normal(0, this parameter)
						lowerCamelCase_				:						Optional[Any]				  =				activation_function

						lowerCamelCase_				:						Union[str, Any]				  =				DecoderConfig(vocab_size=__lowercase					,			bos_token_id=__lowercase   )
						if "decoder" in common_kwargs:
							del common_kwargs["decoder"]

						lowerCamelCase_				:						int				  =				scale_embedding  # scale factor will be sqrt(d_model) if True

						# 3 Types of Dropout
						lowerCamelCase_				:						Optional[int]				  =				attention_dropout
						lowerCamelCase_				:						Union[str, Any]				  =				activation_dropout
						lowerCamelCase_				:						Dict				  =				dropout

						lowerCamelCase_				:						Optional[int]				  =				use_cache
						super().__init__(
						    pad_token_id=__lowercase					,			bos_token_id=__lowercase					,			eos_token_id=__lowercase					,			decoder_start_token_id=__lowercase					,			is_encoder_decoder=__lowercase					,			tie_word_embeddings=__lowercase					,			forced_eos_token_id=__lowercase					,			max_length=__lowercase					,			num_beams=__lowercase					,			length_penalty=__lowercase					,			early_stopping=__lowercase					,			**__lowercase					,			)





					def       lowercase_				(							self   ):







						'''simple docstring'''




						lowerCamelCase_				:						List[Any]				  =				copy.deepcopy(self.__dict__   )
						lowerCamelCase_				:						Optional[int]				  =				self.decoder.to_dict()
						lowerCamelCase_				:						Dict				  =				self.__class__.model_type
						return output
