




'''simple docstring'''







import json
import os
import shutil
import tempfile
import unittest

from transformers import is_speech_available, is_torch_available
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_torch
from transformers.utils import FEATURE_EXTRACTOR_NAME


if is_speech_available() and is_torch_available():
						from transformers import SpeechTaFeatureExtractor, SpeechTaProcessor

						from .test_feature_extraction_speechta import floats_list


_lowerCamelCase				:		List[str]				      =    get_tests_dir('fixtures/test_sentencepiece_bpe_char.model')





@require_torch
class 	lowerCamelCase_				(			unittest.TestCase				):




					'''simple docstring'''






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			tempfile.mkdtemp()

							_snake_case      			=			SpeechTaTokenizer(_lowerCamelCase					)
							tokenizer.save_pretrained(self.tmpdirname					)

							_snake_case      			=			{
							    "feature_size": 1,
							    "padding_value": 0.0,
							    "sampling_rate": 16_000,
							    "do_normalize": False,
							    "num_mel_bins": 80,
							    "hop_length": 16,
							    "win_length": 64,
							    "win_function": "hann_window",
							    "fmin": 80,
							    "fmax": 7_600,
							    "mel_floor": 1E-10,
							    "reduction_factor": 2,
							    "return_attention_mask": True,
							}

							_snake_case      			=			os.path.join(self.tmpdirname,    _lowerCamelCase					)
							with open(self.feature_extraction_file,    "w",    encoding="utf-8"					) as fp:
									fp.write(json.dumps(_lowerCamelCase					) + "\n"					)






					def 							__SCREAMING_SNAKE_CASE      (       self,    **__UpperCamelCase					):
							return SpeechTaTokenizer.from_pretrained(self.tmpdirname,    **_lowerCamelCase					)






					def 							__SCREAMING_SNAKE_CASE      (       self,    **__UpperCamelCase					):
							return SpeechTaFeatureExtractor.from_pretrained(self.tmpdirname,    **_lowerCamelCase					)






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							shutil.rmtree(self.tmpdirname					)






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			self.get_tokenizer()
							_snake_case      			=			self.get_feature_extractor()

							_snake_case      			=			SpeechTaProcessor(tokenizer=_lowerCamelCase,    feature_extractor=_lowerCamelCase					)

							processor.save_pretrained(self.tmpdirname					)
							_snake_case      			=			SpeechTaProcessor.from_pretrained(self.tmpdirname					)

							self.assertEqual(processor.tokenizer.get_vocab(),    tokenizer.get_vocab()					)
							self.assertIsInstance(processor.tokenizer,    _lowerCamelCase					)

							self.assertEqual(processor.feature_extractor.to_json_string(),    feature_extractor.to_json_string()					)
							self.assertIsInstance(processor.feature_extractor,    _lowerCamelCase					)






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			SpeechTaProcessor(tokenizer=self.get_tokenizer(),    feature_extractor=self.get_feature_extractor()					)
							processor.save_pretrained(self.tmpdirname					)

							_snake_case      			=			self.get_tokenizer(bos_token="(BOS)",    eos_token="(EOS)"					)
							_snake_case      			=			self.get_feature_extractor(do_normalize=_lowerCamelCase,    padding_value=1.0					)

							_snake_case      			=			SpeechTaProcessor.from_pretrained(
							    self.tmpdirname,    bos_token="(BOS)",    eos_token="(EOS)",    do_normalize=_lowerCamelCase,    padding_value=1.0					)

							self.assertEqual(processor.tokenizer.get_vocab(),    tokenizer_add_kwargs.get_vocab()					)
							self.assertIsInstance(processor.tokenizer,    _lowerCamelCase					)

							self.assertEqual(processor.feature_extractor.to_json_string(),    feature_extractor_add_kwargs.to_json_string()					)
							self.assertIsInstance(processor.feature_extractor,    _lowerCamelCase					)






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			self.get_feature_extractor()
							_snake_case      			=			self.get_tokenizer()

							_snake_case      			=			SpeechTaProcessor(tokenizer=_lowerCamelCase,    feature_extractor=_lowerCamelCase					)

							_snake_case      			=			floats_list((3, 1_000)					)

							_snake_case      			=			feature_extractor(audio=_lowerCamelCase,    return_tensors="np"					)
							_snake_case      			=			processor(audio=_lowerCamelCase,    return_tensors="np"					)

							for key in input_feat_extract.keys():
									self.assertAlmostEqual(input_feat_extract[key].sum(),    input_processor[key].sum(),    delta=1E-2					)






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			self.get_feature_extractor()
							_snake_case      			=			self.get_tokenizer()

							_snake_case      			=			SpeechTaProcessor(tokenizer=_lowerCamelCase,    feature_extractor=_lowerCamelCase					)

							_snake_case      			=			floats_list((3, 1_000)					)

							_snake_case      			=			feature_extractor(audio_target=_lowerCamelCase,    return_tensors="np"					)
							_snake_case      			=			processor(audio_target=_lowerCamelCase,    return_tensors="np"					)

							for key in input_feat_extract.keys():
									self.assertAlmostEqual(input_feat_extract[key].sum(),    input_processor[key].sum(),    delta=1E-2					)






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			self.get_feature_extractor()
							_snake_case      			=			self.get_tokenizer()

							_snake_case      			=			SpeechTaProcessor(tokenizer=_lowerCamelCase,    feature_extractor=_lowerCamelCase					)

							_snake_case      			=			"This is a test string"

							_snake_case      			=			processor(text=_lowerCamelCase					)
							_snake_case      			=			tokenizer(_lowerCamelCase					)

							for key in encoded_tok.keys():
									self.assertListEqual(encoded_tok[key],    encoded_processor[key]					)






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			self.get_feature_extractor()
							_snake_case      			=			self.get_tokenizer()

							_snake_case      			=			SpeechTaProcessor(tokenizer=_lowerCamelCase,    feature_extractor=_lowerCamelCase					)

							_snake_case      			=			"This is a test string"

							_snake_case      			=			processor(text_target=_lowerCamelCase					)
							_snake_case      			=			tokenizer(_lowerCamelCase					)

							for key in encoded_tok.keys():
									self.assertListEqual(encoded_tok[key],    encoded_processor[key]					)






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			self.get_feature_extractor()
							_snake_case      			=			self.get_tokenizer()

							_snake_case      			=			SpeechTaProcessor(tokenizer=_lowerCamelCase,    feature_extractor=_lowerCamelCase					)

							_snake_case      			=			[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

							_snake_case      			=			processor.batch_decode(_lowerCamelCase					)
							_snake_case      			=			tokenizer.batch_decode(_lowerCamelCase					)

							self.assertListEqual(_lowerCamelCase,    _lowerCamelCase					)






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			self.get_feature_extractor()
							_snake_case      			=			self.get_tokenizer()

							_snake_case      			=			SpeechTaProcessor(tokenizer=_lowerCamelCase,    feature_extractor=_lowerCamelCase					)

							self.assertListEqual(
							    processor.model_input_names,    feature_extractor.model_input_names,    msg="`processor` and `feature_extractor` model input names do not match",    )













"""simple docstring"""


from typing import TYPE_CHECKING

from ..utils import OptionalDependencyNotAvailable, _LazyModule, is_flax_available, is_tf_available, is_torch_available


lowercase_								=	{
    'configuration_utils': ['GenerationConfig'],
    'streamers': ['TextIteratorStreamer', 'TextStreamer'],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowercase_								=	[
           'Constraint',
           'ConstraintListState',
           'DisjunctiveConstraint',
           'PhrasalConstraint',
       ]
       lowercase_								=	[
           'BeamHypotheses',
           'BeamScorer',
           'BeamSearchScorer',
           'ConstrainedBeamSearchScorer',
       ]
       lowercase_								=	[
           'EpsilonLogitsWarper',
           'EtaLogitsWarper',
           'ForcedBOSTokenLogitsProcessor',
           'ForcedEOSTokenLogitsProcessor',
           'HammingDiversityLogitsProcessor',
           'InfNanRemoveLogitsProcessor',
           'LogitsProcessor',
           'LogitsProcessorList',
           'LogitsWarper',
           'MinLengthLogitsProcessor',
           'MinNewTokensLengthLogitsProcessor',
           'NoBadWordsLogitsProcessor',
           'NoRepeatNGramLogitsProcessor',
           'PrefixConstrainedLogitsProcessor',
           'RepetitionPenaltyLogitsProcessor',
           'SequenceBiasLogitsProcessor',
           'EncoderRepetitionPenaltyLogitsProcessor',
           'TemperatureLogitsWarper',
           'TopKLogitsWarper',
           'TopPLogitsWarper',
           'TypicalLogitsWarper',
           'EncoderNoRepeatNGramLogitsProcessor',
           'ExponentialDecayLengthPenalty',
           'LogitNormalization',
       ]
       lowercase_								=	[
           'MaxNewTokensCriteria',
           'MaxLengthCriteria',
           'MaxTimeCriteria',
           'StoppingCriteria',
           'StoppingCriteriaList',
           'validate_stopping_criteria',
       ]
       lowercase_								=	[
           'GenerationMixin',
           'top_k_top_p_filtering',
           'GreedySearchEncoderDecoderOutput',
           'GreedySearchDecoderOnlyOutput',
           'SampleEncoderDecoderOutput',
           'SampleDecoderOnlyOutput',
           'BeamSearchEncoderDecoderOutput',
           'BeamSearchDecoderOnlyOutput',
           'BeamSampleEncoderDecoderOutput',
           'BeamSampleDecoderOnlyOutput',
           'ContrastiveSearchEncoderDecoderOutput',
           'ContrastiveSearchDecoderOnlyOutput',
       ]

try:
       if not is_tf_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowercase_								=	[
           'TFForcedBOSTokenLogitsProcessor',
           'TFForcedEOSTokenLogitsProcessor',
           'TFLogitsProcessor',
           'TFLogitsProcessorList',
           'TFLogitsWarper',
           'TFMinLengthLogitsProcessor',
           'TFNoBadWordsLogitsProcessor',
           'TFNoRepeatNGramLogitsProcessor',
           'TFRepetitionPenaltyLogitsProcessor',
           'TFTemperatureLogitsWarper',
           'TFTopKLogitsWarper',
           'TFTopPLogitsWarper',
           'TFForceTokensLogitsProcessor',
           'TFSuppressTokensAtBeginLogitsProcessor',
           'TFSuppressTokensLogitsProcessor',
       ]
       lowercase_								=	[
           'TFGenerationMixin',
           'tf_top_k_top_p_filtering',
           'TFGreedySearchDecoderOnlyOutput',
           'TFGreedySearchEncoderDecoderOutput',
           'TFSampleEncoderDecoderOutput',
           'TFSampleDecoderOnlyOutput',
           'TFBeamSearchEncoderDecoderOutput',
           'TFBeamSearchDecoderOnlyOutput',
           'TFBeamSampleEncoderDecoderOutput',
           'TFBeamSampleDecoderOnlyOutput',
           'TFContrastiveSearchEncoderDecoderOutput',
           'TFContrastiveSearchDecoderOnlyOutput',
       ]

try:
       if not is_flax_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowercase_								=	[
           'FlaxForcedBOSTokenLogitsProcessor',
           'FlaxForcedEOSTokenLogitsProcessor',
           'FlaxLogitsProcessor',
           'FlaxLogitsProcessorList',
           'FlaxLogitsWarper',
           'FlaxMinLengthLogitsProcessor',
           'FlaxTemperatureLogitsWarper',
           'FlaxTopKLogitsWarper',
           'FlaxTopPLogitsWarper',
       ]
       lowercase_								=	[
           'FlaxGenerationMixin',
           'FlaxGreedySearchOutput',
           'FlaxSampleOutput',
           'FlaxBeamSearchOutput',
       ]

if TYPE_CHECKING:
       from .configuration_utils import GenerationConfig
       from .streamers import TextIteratorStreamer, TextStreamer

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .beam_constraints import Constraint, ConstraintListState, DisjunctiveConstraint, PhrasalConstraint
              from .beam_search import BeamHypotheses, BeamScorer, BeamSearchScorer, ConstrainedBeamSearchScorer
              from .logits_process import (
                  EncoderNoRepeatNGramLogitsProcessor,
                  EncoderRepetitionPenaltyLogitsProcessor,
                  EpsilonLogitsWarper,
                  EtaLogitsWarper,
                  ExponentialDecayLengthPenalty,
                  ForcedBOSTokenLogitsProcessor,
                  ForcedEOSTokenLogitsProcessor,
                  HammingDiversityLogitsProcessor,
                  InfNanRemoveLogitsProcessor,
                  LogitNormalization,
                  LogitsProcessor,
                  LogitsProcessorList,
                  LogitsWarper,
                  MinLengthLogitsProcessor,
                  MinNewTokensLengthLogitsProcessor,
                  NoBadWordsLogitsProcessor,
                  NoRepeatNGramLogitsProcessor,
                  PrefixConstrainedLogitsProcessor,
                  RepetitionPenaltyLogitsProcessor,
                  SequenceBiasLogitsProcessor,
                  TemperatureLogitsWarper,
                  TopKLogitsWarper,
                  TopPLogitsWarper,
                  TypicalLogitsWarper,
              )
              from .stopping_criteria import (
                  MaxLengthCriteria,
                  MaxNewTokensCriteria,
                  MaxTimeCriteria,
                  StoppingCriteria,
                  StoppingCriteriaList,
                  validate_stopping_criteria,
              )
              from .utils import (
                  BeamSampleDecoderOnlyOutput,
                  BeamSampleEncoderDecoderOutput,
                  BeamSearchDecoderOnlyOutput,
                  BeamSearchEncoderDecoderOutput,
                  ContrastiveSearchDecoderOnlyOutput,
                  ContrastiveSearchEncoderDecoderOutput,
                  GenerationMixin,
                  GreedySearchDecoderOnlyOutput,
                  GreedySearchEncoderDecoderOutput,
                  SampleDecoderOnlyOutput,
                  SampleEncoderDecoderOutput,
                  top_k_top_p_filtering,
              )

       try:
              if not is_tf_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .tf_logits_process import (
                  TFForcedBOSTokenLogitsProcessor,
                  TFForcedEOSTokenLogitsProcessor,
                  TFForceTokensLogitsProcessor,
                  TFLogitsProcessor,
                  TFLogitsProcessorList,
                  TFLogitsWarper,
                  TFMinLengthLogitsProcessor,
                  TFNoBadWordsLogitsProcessor,
                  TFNoRepeatNGramLogitsProcessor,
                  TFRepetitionPenaltyLogitsProcessor,
                  TFSuppressTokensAtBeginLogitsProcessor,
                  TFSuppressTokensLogitsProcessor,
                  TFTemperatureLogitsWarper,
                  TFTopKLogitsWarper,
                  TFTopPLogitsWarper,
              )
              from .tf_utils import (
                  TFBeamSampleDecoderOnlyOutput,
                  TFBeamSampleEncoderDecoderOutput,
                  TFBeamSearchDecoderOnlyOutput,
                  TFBeamSearchEncoderDecoderOutput,
                  TFContrastiveSearchDecoderOnlyOutput,
                  TFContrastiveSearchEncoderDecoderOutput,
                  TFGenerationMixin,
                  TFGreedySearchDecoderOnlyOutput,
                  TFGreedySearchEncoderDecoderOutput,
                  TFSampleDecoderOnlyOutput,
                  TFSampleEncoderDecoderOutput,
                  tf_top_k_top_p_filtering,
              )

       try:
              if not is_flax_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .flax_logits_process import (
                  FlaxForcedBOSTokenLogitsProcessor,
                  FlaxForcedEOSTokenLogitsProcessor,
                  FlaxLogitsProcessor,
                  FlaxLogitsProcessorList,
                  FlaxLogitsWarper,
                  FlaxMinLengthLogitsProcessor,
                  FlaxTemperatureLogitsWarper,
                  FlaxTopKLogitsWarper,
                  FlaxTopPLogitsWarper,
              )
              from .flax_utils import FlaxBeamSearchOutput, FlaxGenerationMixin, FlaxGreedySearchOutput, FlaxSampleOutput
else:
       import sys

       lowercase_								=	_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




