




'''simple docstring'''




import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
     from sagemaker import Session, TrainingJobAnalytics
     from sagemaker.huggingface import HuggingFace


@pytest.mark.skipif(
    literal_eval(os.getenv('TEST_SAGEMAKER'			,					'False'				)				) is not True			,					reason='Skipping test because should only be run when releasing minor transformers version'			,					)
@pytest.mark.usefixtures('sm_env'				)
@parameterized_class(
    [
        {
            'framework': 'pytorch',
            'script': 'run_glue.py',
            'model_name_or_path': 'distilbert-base-cased',
            'instance_type': 'ml.p3.16xlarge',
            'results': {'train_runtime': 6_50, 'eval_accuracy': 0.7, 'eval_loss': 0.6},
        },
        {
            'framework': 'pytorch',
            'script': 'run_ddp.py',
            'model_name_or_path': 'distilbert-base-cased',
            'instance_type': 'ml.p3.16xlarge',
            'results': {'train_runtime': 6_00, 'eval_accuracy': 0.7, 'eval_loss': 0.6},
        },
        {
            'framework': 'tensorflow',
            'script': 'run_tf_dist.py',
            'model_name_or_path': 'distilbert-base-cased',
            'instance_type': 'ml.p3.16xlarge',
            'results': {'train_runtime': 6_00, 'eval_accuracy': 0.6, 'eval_loss': 0.7},
        },
    ]				)
class   __UpperCamelCase       ( unittest.TestCase				):






       def      lowercase__     (							self  ):



              """simple docstring"""







              if self.framework == "pytorch":
                     subprocess.run(
                         f'''cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py'''.split(),		encoding='''utf-8''',		check=lowerCAmelCase,		)
              assert hasattr(self,		'''env'''  )






       def      lowercase__     (							self,		lowerCAmelCase  ):



              """simple docstring"""







              lowerCamelCase_							=f'''{self.env.base_job_name}-{instance_count}-{'ddp' if 'ddp' in self.script else 'smd'}'''
              # distributed data settings
              lowerCamelCase_							={'''smdistributed''': {'''dataparallel''': {'''enabled''': True}}} if self.script != '''run_ddp.py''' else None

              # creates estimator
              return HuggingFace(
                  entry_point=self.script,		source_dir=self.env.test_path,		role=self.env.role,		image_uri=self.env.image_uri,		base_job_name=lowerCAmelCase,		instance_count=lowerCAmelCase,		instance_type=self.instance_type,		debugger_hook_config=lowerCAmelCase,		hyperparameters={**self.env.distributed_hyperparameters, '''model_name_or_path''': self.model_name_or_path},		metric_definitions=self.env.metric_definitions,		distribution=lowerCAmelCase,		py_version='''py36''',		)






       def      lowercase__     (							self,		lowerCAmelCase  ):



              """simple docstring"""







              TrainingJobAnalytics(lowerCAmelCase  ).export_csv(f'''{self.env.test_path}/{job_name}_metrics.csv'''  )






       @parameterized.expand([(2,)]  )
       def      lowercase__     (							self,		lowerCAmelCase  ):



              """simple docstring"""







              lowerCamelCase_							=self.create_estimator(lowerCAmelCase  )

              # run training
              estimator.fit()

              # result dataframe
              lowerCamelCase_							=TrainingJobAnalytics(estimator.latest_training_job.name  ).dataframe()

              # extract kpis
              lowerCamelCase_							=list(result_metrics_df[result_metrics_df.metric_name == '''eval_accuracy''']['''value''']  )
              lowerCamelCase_							=list(result_metrics_df[result_metrics_df.metric_name == '''eval_loss''']['''value''']  )
              # get train time from SageMaker job, this includes starting, preprocessing, stopping
              lowerCamelCase_							=(
                  Session().describe_training_job(estimator.latest_training_job.name  ).get('''TrainingTimeInSeconds''',		999_999  )
              )

              # assert kpis
              assert train_runtime <= self.results["train_runtime"]
              assert all(t >= self.results['''eval_accuracy'''] for t in eval_accuracy  )
              assert all(t <= self.results['''eval_loss'''] for t in eval_loss  )

              # dump tests result into json file to share in PR
              with open(f'''{estimator.latest_training_job.name}.json''',		'''w'''  ) as outfile:
                     json.dump({'''train_time''': train_runtime, '''eval_accuracy''': eval_accuracy, '''eval_loss''': eval_loss},		lowerCAmelCase  )






'''simple docstring'''




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_speech_available,
    is_tf_available,
    is_torch_available,
)


lowercase			:				Dict    		=			{
    'configuration_speech_to_text': ['SPEECH_TO_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'Speech2TextConfig'],
    'processing_speech_to_text': ['Speech2TextProcessor'],
}

try:
      if not is_sentencepiece_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      lowercase			:				List[Any]    		=			['Speech2TextTokenizer']

try:
      if not is_speech_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      lowercase			:				List[str]    		=			['Speech2TextFeatureExtractor']

try:
      if not is_tf_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      lowercase			:				Dict    		=			[
          'TF_SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST',
          'TFSpeech2TextForConditionalGeneration',
          'TFSpeech2TextModel',
          'TFSpeech2TextPreTrainedModel',
      ]

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      lowercase			:				Any    		=			[
          'SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST',
          'Speech2TextForConditionalGeneration',
          'Speech2TextModel',
          'Speech2TextPreTrainedModel',
      ]


if TYPE_CHECKING:
      from .configuration_speech_to_text import SPEECH_TO_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP, SpeechaTextConfig
      from .processing_speech_to_text import SpeechaTextProcessor

      try:
            if not is_sentencepiece_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .tokenization_speech_to_text import SpeechaTextTokenizer

      try:
            if not is_speech_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .feature_extraction_speech_to_text import SpeechaTextFeatureExtractor

      try:
            if not is_tf_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_tf_speech_to_text import (
                TF_SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
                TFSpeechaTextForConditionalGeneration,
                TFSpeechaTextModel,
                TFSpeechaTextPreTrainedModel,
            )

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_speech_to_text import (
                SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
                SpeechaTextForConditionalGeneration,
                SpeechaTextModel,
                SpeechaTextPreTrainedModel,
            )

else:
      import sys

      lowercase			:				List[Any]    		=			_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




