






"""simple docstring"""
import random
import timeit
from functools import wraps
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_tf_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_tf_available():
			import tensorflow as tf
			from tensorflow.python.framework.errors_impl import ResourceExhaustedError

			from .benchmark_args_tf import TensorFlowBenchmarkArguments

if is_pyanvml_available():
			import pyanvml.pyanvml as nvml

lowerCamelCase_					=      logging.get_logger(__name__)

def 				snake_case      (   A__	,A__						):


			def run_func(A__						):
						@wraps(__A						)
						def run_in_eager_mode(*A__	,**A__						):
									return func(*__A	,**__A						)

						@wraps(__A						)
						@tf.function(experimental_compile=__A						)
						def run_in_graph_mode(*A__	,**A__						):
									return func(*__A	,**__A						)

						if do_eager_mode is True:
									if use_xla is not False:
												raise ValueError(
												    "Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`."						)
									return run_in_eager_mode
						else:
									return run_in_graph_mode

			return run_func

def 				snake_case      (   A__	,A__	,A__						):
			UpperCAmelCase_     : Tuple			     =					random.Random()
			UpperCAmelCase_     : Optional[Any]			     =					[rng.randint(0	,vocab_size - 1						) for i in range(batch_size * sequence_length						)]
			return tf.constant(__A	,shape=(batch_size, sequence_length)	,dtype=tf.intaa						)







class 			UpperCamelCase_       (UpperCAmelCase_     ):
	__magic_name__	      =				42
	__magic_name__	      =				42
	__magic_name__	      =				"TensorFlow"





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: Any						)						->			str:
				return tf.__version__





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]	,	lowerCAmelCase_					: str	,	lowerCAmelCase_					: int	,	lowerCAmelCase_					: int						)						->			float:
				UpperCAmelCase_     : Union[str, Any]			     =					self.args.strategy
				if strategy is None:
							raise ValueError("A device strategy has to be initialized before using TensorFlow."						)
				UpperCAmelCase_     : Tuple			     =					self._prepare_inference_func(__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase						)
				return self._measure_speed(_inference						)





	def      _SCREAMING_SNAKE_CASE      (      self					: str	,	lowerCAmelCase_					: str	,	lowerCAmelCase_					: int	,	lowerCAmelCase_					: int						)						->			float:
				UpperCAmelCase_     : Any			     =					self.args.strategy
				if strategy is None:
							raise ValueError("A device strategy has to be initialized before using TensorFlow."						)
				UpperCAmelCase_     : int			     =					self._prepare_train_func(__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase						)
				return self._measure_speed(_train						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Tuple	,	lowerCAmelCase_					: str	,	lowerCAmelCase_					: int	,	lowerCAmelCase_					: int						)						->			[Memory, Optional[MemorySummary]]:



				if self.args.is_gpu:
							tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]	,	__UpperCAmelCase						)
				UpperCAmelCase_     : List[Any]			     =					self.args.strategy
				if strategy is None:
							raise ValueError("A device strategy has to be initialized before using TensorFlow."						)
				UpperCAmelCase_     : Optional[int]			     =					self._prepare_inference_func(__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase						)
				return self._measure_memory(_inference						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Dict	,	lowerCAmelCase_					: str	,	lowerCAmelCase_					: int	,	lowerCAmelCase_					: int						)						->			[Memory, Optional[MemorySummary]]:



				if self.args.is_gpu:
							tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]	,	__UpperCAmelCase						)
				UpperCAmelCase_     : List[Any]			     =					self.args.strategy
				if strategy is None:
							raise ValueError("A device strategy has to be initialized before using TensorFlow."						)

				UpperCAmelCase_     : Optional[Any]			     =					self._prepare_train_func(__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase						)
				return self._measure_memory(_train						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[int]	,	lowerCAmelCase_					: str	,	lowerCAmelCase_					: int	,	lowerCAmelCase_					: int						)						->			Callable[[], None]:
				UpperCAmelCase_     : Dict			     =					self.config_dict[model_name]

				if self.args.fpaa:
							raise NotImplementedError("Mixed precision is currently not supported."						)

				UpperCAmelCase_     : int			     =					(
				    hasattr(__UpperCAmelCase	,	"architectures"						)
				    and isinstance(config.architectures	,	__UpperCAmelCase						)
				    and len(config.architectures						) > 0
				)
				if not self.args.only_pretrain_model and has_model_class_in_config:
							try:
										UpperCAmelCase_     : List[Any]			     =					"TF" + config.architectures[0]  # prepend 'TF' for tensorflow model
										UpperCAmelCase_     : Optional[int]			     =					__import__("transformers"	,	fromlist=[model_class]						)
										UpperCAmelCase_     : Dict			     =					getattr(__UpperCAmelCase	,	__UpperCAmelCase						)
										UpperCAmelCase_     : str			     =					model_cls(__UpperCAmelCase						)
							except ImportError:
										raise ImportError(
										    f"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
										    " set `--only_pretrain_model` or `args.only_pretrain_model=True`."						)
				else:
							UpperCAmelCase_     : List[str]			     =					TF_MODEL_MAPPING[config.__class__](__UpperCAmelCase						)

				# encoder-decoder has vocab size saved differently
				UpperCAmelCase_     : List[str]			     =					config.vocab_size if hasattr(__UpperCAmelCase	,	"vocab_size"						) else config.encoder.vocab_size
				UpperCAmelCase_     : Any			     =					random_input_ids(__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase						)

				@run_with_tf_optimizations(self.args.eager_mode	,	self.args.use_xla						)
				def encoder_decoder_forward():
							return model(__UpperCAmelCase	,	decoder_input_ids=__UpperCAmelCase	,	training=__UpperCAmelCase						)

				@run_with_tf_optimizations(self.args.eager_mode	,	self.args.use_xla						)
				def encoder_forward():
							return model(__UpperCAmelCase	,	training=__UpperCAmelCase						)

				UpperCAmelCase_     : Tuple			     =					encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

				return _inference





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]	,	lowerCAmelCase_					: str	,	lowerCAmelCase_					: int	,	lowerCAmelCase_					: int						)						->			Callable[[], None]:
				UpperCAmelCase_     : Union[str, Any]			     =					self.config_dict[model_name]

				if self.args.eager_mode is not False:
							raise ValueError("Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`."						)

				if self.args.fpaa:
							raise NotImplementedError("Mixed precision is currently not supported."						)

				UpperCAmelCase_     : int			     =					(
				    hasattr(__UpperCAmelCase	,	"architectures"						)
				    and isinstance(config.architectures	,	__UpperCAmelCase						)
				    and len(config.architectures						) > 0
				)
				if not self.args.only_pretrain_model and has_model_class_in_config:
							try:
										UpperCAmelCase_     : Optional[Any]			     =					"TF" + config.architectures[0]  # prepend 'TF' for tensorflow model
										UpperCAmelCase_     : Dict			     =					__import__("transformers"	,	fromlist=[model_class]						)
										UpperCAmelCase_     : List[Any]			     =					getattr(__UpperCAmelCase	,	__UpperCAmelCase						)
										UpperCAmelCase_     : List[Any]			     =					model_cls(__UpperCAmelCase						)
							except ImportError:
										raise ImportError(
										    f"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
										    " set `--only_pretrain_model` or `args.only_pretrain_model=True`."						)
				else:
							UpperCAmelCase_     : List[str]			     =					TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](__UpperCAmelCase						)

				# encoder-decoder has vocab size saved differently
				UpperCAmelCase_     : List[Any]			     =					config.vocab_size if hasattr(__UpperCAmelCase	,	"vocab_size"						) else config.encoder.vocab_size
				UpperCAmelCase_     : int			     =					random_input_ids(__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase						)

				@run_with_tf_optimizations(self.args.eager_mode	,	self.args.use_xla						)
				def encoder_decoder_train():
							UpperCAmelCase_     : Any			     =					model(__UpperCAmelCase	,	decoder_input_ids=__UpperCAmelCase	,	labels=__UpperCAmelCase	,	training=__UpperCAmelCase						)[0]
							UpperCAmelCase_     : int			     =					tf.gradients(__UpperCAmelCase	,	model.trainable_variables						)
							return gradients

				@run_with_tf_optimizations(self.args.eager_mode	,	self.args.use_xla						)
				def encoder_train():
							UpperCAmelCase_     : str			     =					model(__UpperCAmelCase	,	labels=__UpperCAmelCase	,	training=__UpperCAmelCase						)[0]
							UpperCAmelCase_     : str			     =					tf.gradients(__UpperCAmelCase	,	model.trainable_variables						)
							return gradients

				UpperCAmelCase_     : Tuple			     =					encoder_decoder_train if config.is_encoder_decoder else encoder_train

				return _train





	def      _SCREAMING_SNAKE_CASE      (      self					: Tuple	,	lowerCAmelCase_					: Dict						)						->			float:



				with self.args.strategy.scope():
							try:
										if self.args.is_tpu or self.args.use_xla:
													# run additional 10 times to stabilize compilation for tpu
													logger.info("Do inference on TPU. Running model 5 times to stabilize compilation"						)
													timeit.repeat(__UpperCAmelCase	,	repeat=1	,	number=5						)

										# as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
										UpperCAmelCase_     : Any			     =					timeit.repeat(
										    __UpperCAmelCase	,	repeat=self.args.repeat	,	number=10	,	)

										return min(__UpperCAmelCase						) / 1_0.0
							except ResourceExhaustedError as e:
										self.print_fn(f"""Doesn't fit on GPU. {e}"""						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]	,	lowerCAmelCase_					: Callable[[], None]						)						->			[Memory, MemorySummary]:
				logger.info(
				    "Note that TensorFlow allocates more memory than "
				    "it might need to speed up computation. "
				    "The memory reported here corresponds to the memory "
				    "reported by `nvidia-smi`, which can vary depending "
				    "on total available memory on the GPU that is used."						)
				with self.args.strategy.scope():
							try:
										if self.args.trace_memory_line_by_line:
													if not self.args.eager_mode:
																raise ValueError(
																    "`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory"
																    " consumption line by line."						)
													UpperCAmelCase_     : List[Any]			     =					start_memory_tracing("transformers"						)

										if self.args.is_tpu:
													# tpu
													raise NotImplementedError(
													    "Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking"
													    " with `args.memory=False`"						)
										elif self.args.is_gpu:
													# gpu
													if not is_pyanvml_available():
																logger.warning(
																    "py3nvml not installed, we won't log GPU memory usage. "
																    "Install py3nvml (pip install py3nvml) to log information about GPU."						)
																UpperCAmelCase_     : Tuple			     =					"N/A"
													else:
																logger.info(
																    "Measuring total GPU usage on GPU device. Make sure to not have additional processes"
																    " running on the same GPU."						)
																# init nvml
																nvml.nvmlInit()
																func()
																UpperCAmelCase_     : Tuple			     =					nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx						)
																UpperCAmelCase_     : int			     =					nvml.nvmlDeviceGetMemoryInfo(__UpperCAmelCase						)
																UpperCAmelCase_     : Any			     =					meminfo.used
																UpperCAmelCase_     : Tuple			     =					Memory(__UpperCAmelCase						)
																# shutdown nvml
																nvml.nvmlShutdown()
										else:
													# cpu
													if self.args.trace_memory_line_by_line:
																logger.info(
																    "When enabling line by line tracing, the max peak memory for CPU is inaccurate in"
																    " TensorFlow."						)
																UpperCAmelCase_     : Dict			     =					None
													else:
																UpperCAmelCase_     : Tuple			     =					measure_peak_memory_cpu(__UpperCAmelCase						)
																UpperCAmelCase_     : Optional[Any]			     =					Memory(__UpperCAmelCase						) if isinstance(__UpperCAmelCase	,	__UpperCAmelCase						) else memory_bytes
										if self.args.trace_memory_line_by_line:
													UpperCAmelCase_     : List[Any]			     =					stop_memory_tracing(__UpperCAmelCase						)
													if memory is None:
																UpperCAmelCase_     : Union[str, Any]			     =					summary.total
										else:
													UpperCAmelCase_     : str			     =					None

										return memory, summary
							except ResourceExhaustedError as e:
										self.print_fn(f"""Doesn't fit on GPU. {e}"""						)
										return "N/A", None




import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
      from sagemaker import Session, TrainingJobAnalytics
      from sagemaker.huggingface import HuggingFace





@pytest.mark.skipif(
    literal_eval(os.getenv('TEST_SAGEMAKER'		,				'False'     )     ) is not True		,				reason='Skipping test because should only be run when releasing minor transformers version'		,				)
@pytest.mark.usefixtures('sm_env'     )
@parameterized_class(
    [
        {
            'framework': 'pytorch',
            'script': 'run_glue.py',
            'model_name_or_path': 'distilbert-base-cased',
            'instance_type': 'ml.g4dn.xlarge',
            'results': {'train_runtime': 6_50, 'eval_accuracy': 0.6, 'eval_loss': 0.9},
        },
        {
            'framework': 'tensorflow',
            'script': 'run_tf.py',
            'model_name_or_path': 'distilbert-base-cased',
            'instance_type': 'ml.g4dn.xlarge',
            'results': {'train_runtime': 6_00, 'eval_accuracy': 0.3, 'eval_loss': 0.9},
        },
    ]     )
class      A							(				unittest.TestCase     ):


   def 			lowercase_      (self     :						int		)						->				Optional[Any]:
         """simple docstring"""



         if self.framework == "pytorch":
               subprocess.run(
                   f"""cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py""".split()       ,  encoding="utf-8"       ,  check=__UpperCAmelCase       ,  )
         assert hasattr(self       ,  "env"		)


   def 			lowercase_      (self     :						List[Any]       ,  __UpperCAmelCase     :						Optional[int]=1		)						->				Dict:
         """simple docstring"""



         return HuggingFace(
             entry_point=self.script       ,  source_dir=self.env.test_path       ,  role=self.env.role       ,  image_uri=self.env.image_uri       ,  base_job_name=f"""{self.env.base_job_name}-single"""       ,  instance_count=__UpperCAmelCase       ,  instance_type=self.instance_type       ,  debugger_hook_config=__UpperCAmelCase       ,  hyperparameters={**self.env.hyperparameters, "model_name_or_path": self.model_name_or_path}       ,  metric_definitions=self.env.metric_definitions       ,  py_version="py36"       ,  )


   def 			lowercase_      (self     :						Optional[Any]       ,  __UpperCAmelCase     :						Tuple		)						->				Optional[int]:
         """simple docstring"""



         TrainingJobAnalytics(__UpperCAmelCase		).export_csv(f"""{self.env.test_path}/{job_name}_metrics.csv"""		)





   def 			lowercase_      (self     :						Any		)						->				Optional[Any]:
         """simple docstring"""



         UpperCAmelCase__								=  self.create_estimator()

         # run training
         estimator.fit()

         # result dataframe
         UpperCAmelCase__								=  TrainingJobAnalytics(estimator.latest_training_job.name		).dataframe()

         # extract kpis
         UpperCAmelCase__								=  list(result_metrics_df[result_metrics_df.metric_name == "eval_accuracy"]["value"]		)
         UpperCAmelCase__								=  list(result_metrics_df[result_metrics_df.metric_name == "eval_loss"]["value"]		)
         # get train time from SageMaker job, this includes starting, preprocessing, stopping
         UpperCAmelCase__								=  (
             Session().describe_training_job(estimator.latest_training_job.name		).get("TrainingTimeInSeconds"       ,  9_9_9_9_9_9		)
         )

         # assert kpis
         assert train_runtime <= self.results["train_runtime"]
         assert all(t >= self.results["eval_accuracy"] for t in eval_accuracy		)
         assert all(t <= self.results["eval_loss"] for t in eval_loss		)

         # dump tests result into json file to share in PR
         with open(f"""{estimator.latest_training_job.name}.json"""       ,  "w"		) as outfile:
               json.dump({"train_time": train_runtime, "eval_accuracy": eval_accuracy, "eval_loss": eval_loss}       ,  __UpperCAmelCase		)



