

"""simple docstring"""






import inspect
import warnings

import torch

from .state import AcceleratorState, GradientState
from .utils import DistributedType, honor_type, is_tpu_available


if is_tpu_available(check_device=False):
							import torch_xla.core.xla_model as xm
def 		A_	(   __UpperCamelCase     :				Union[str, Any]       ,	__UpperCamelCase     :				Optional[int]				):
		if isinstance(__UpperCamelCase       ,	(list, tuple)				):
				return honor_type(__UpperCamelCase       ,	(move_to_device(__UpperCamelCase       ,	__UpperCamelCase				) for t in state)				)
		elif isinstance(__UpperCamelCase       ,	__UpperCamelCase				):
				return type(__UpperCamelCase				)({k: move_to_device(__UpperCamelCase       ,	__UpperCamelCase				) for k, v in state.items()}				)
		elif isinstance(__UpperCamelCase       ,	torch.Tensor				):
				return state.to(__UpperCamelCase				)
		return state



class 	_lowerCAmelCase    (      torch.optim.Optimizer       ):

							def __init__( self       :    int ,		a       :    List[str] ,		a       :    Optional[int]=True ,		a       :    Optional[Any]=None    )   ->    Optional[Any]:

									"""simple docstring"""
									lowercase										=							optimizer
									lowercase										=							scaler
									lowercase										=							AcceleratorState()
									lowercase										=							GradientState()
									lowercase										=							device_placement
									lowercase										=							False
									lowercase										=							None

									# Handle device placement
									if device_placement:
											lowercase										=							self.optimizer.state_dict()
											if self.accelerator_state.distributed_type == DistributedType.TPU:
													xm.send_cpu_data_to_device(a ,		self.accelerator_state.device    )
											else:
													lowercase										=							move_to_device(a ,		self.accelerator_state.device    )
											self.optimizer.load_state_dict(a    )

							@property
							def 					_lowerCAmelCase  ( self       :    Optional[Any]    )   ->    Any:

									"""simple docstring"""
									return self.optimizer.state

							@state.setter
							def 					_lowerCAmelCase  ( self       :    int ,		a       :    int    )   ->    List[str]:

									"""simple docstring"""
									lowercase										=							state

							@property
							def 					_lowerCAmelCase  ( self       :    Optional[Any]    )   ->    str:

									"""simple docstring"""
									return self.optimizer.param_groups

							@param_groups.setter
							def 					_lowerCAmelCase  ( self       :    List[str] ,		a       :    Tuple    )   ->    Tuple:

									"""simple docstring"""
									lowercase										=							param_groups

							@property
							def 					_lowerCAmelCase  ( self       :    List[str]    )   ->    List[Any]:

									"""simple docstring"""
									return self.optimizer.defaults

							@defaults.setter
							def 					_lowerCAmelCase  ( self       :    Optional[Any] ,		a       :    Any    )   ->    Optional[int]:

									"""simple docstring"""
									lowercase										=							defaults

							def 					_lowerCAmelCase  ( self       :    List[str] ,		a       :    int    )   ->    Union[str, Any]:

									"""simple docstring"""
									self.optimizer.add_param_group(a    )

							def 					_lowerCAmelCase  ( self       :    List[str] ,		a       :    List[str]    )   ->    Tuple:

									"""simple docstring"""
									if self.accelerator_state.distributed_type == DistributedType.TPU and self.device_placement:
											xm.send_cpu_data_to_device(a ,		self.accelerator_state.device    )
									self.optimizer.load_state_dict(a    )

							def 					_lowerCAmelCase  ( self       :    Tuple    )   ->    Union[str, Any]:

									"""simple docstring"""
									return self.optimizer.state_dict()

							def 					_lowerCAmelCase  ( self       :    Optional[int] ,		a       :    Optional[Any]=None    )   ->    int:

									"""simple docstring"""
									if self.gradient_state.sync_gradients:
											lowercase										=							'''set_to_none''' in inspect.signature(self.optimizer.zero_grad    ).parameters
											if accept_arg:
													if set_to_none is None:
															lowercase										=							False
													self.optimizer.zero_grad(set_to_none=a    )
											else:
													if set_to_none is not None:
															raise ValueError('''`set_to_none` for Optimizer.zero_grad` is not supported by this optimizer.'''    )
													self.optimizer.zero_grad()

							def 					_lowerCAmelCase  ( self       :    Dict ,		a       :    Tuple=None    )   ->    Any:

									"""simple docstring"""
									if self.gradient_state.sync_gradients:
											if self.accelerator_state.distributed_type == DistributedType.TPU:
													lowercase										=							{'''closure''': closure} if closure is not None else {}
													xm.optimizer_step(self.optimizer ,		optimizer_args=a    )
											elif self.scaler is not None:
													lowercase										=							False
													if self._last_scale is None:
															# `get_scale` is an async operation requiring full synchronization
															# on CPU and GPUs before finishing. As a result, we store away
															# the prior one to reduce the call overhead
															lowercase										=							self.scaler.get_scale()
															lowercase										=							True
													self.scaler.step(self.optimizer ,		a    )
													self.scaler.update()
													lowercase										=							self.scaler.get_scale()
													if not new_scale:
															# If we reduced the loss scale, it means the optimizer step was skipped because of gradient overflow.
															lowercase										=							scale_after < self._last_scale
													lowercase										=							scale_after

											else:
													self.optimizer.step(a    )

							def 					_lowerCAmelCase  ( self       :    Tuple ,		a       :    Union[str, Any]    )   ->    List[Any]:

									"""simple docstring"""
									for param_group in self.optimizer.param_groups:
											lowercase										=							[parameters_map.get(a ,		a    ) for p in param_group['''params''']]

							@property
							def 					_lowerCAmelCase  ( self       :    List[Any]    )   ->    str:

									"""simple docstring"""
									warnings.warn(
									    '''The `is_overflow` property is deprecated and will be removed in version 1.0 of Accelerate use '''
									    '''`optimizer.step_was_skipped` instead.''' ,		a ,		)
									return self._is_overflow

							@property
							def 					_lowerCAmelCase  ( self       :    Optional[Any]    )   ->    Optional[int]:

									"""simple docstring"""
									return self._is_overflow

							def __getstate__( self       :    str    )   ->    int:

									"""simple docstring"""
									return self.__dict__.copy()

							def __setstate__( self       :    Any ,		a       :    int    )   ->    Dict:

									"""simple docstring"""
									self.__dict__.update(a    )


"""simple docstring"""






import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
							import torch

							from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
							import pyanvml.pyanvml as nvml


__lowerCAmelCase				      =   logging.get_logger(__name__)
class 	_lowerCAmelCase    (      __snake_case       ):
							__lowerCAmelCase					:    PyTorchBenchmarkArguments
							__lowerCAmelCase					:    PretrainedConfig
							__lowerCAmelCase					:    str     =  "PyTorch"

							@property
							def 					_lowerCAmelCase  ( self       :    Dict    )   ->    Dict:

									"""simple docstring"""
									return torch.__version__

							def 					_lowerCAmelCase  ( self       :    List[Any] ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    float:

									"""simple docstring"""
									lowercase										=							self._prepare_inference_func(a ,		a ,		a    )
									return self._measure_speed(_inference    )

							def 					_lowerCAmelCase  ( self       :    str ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    [Memory, Optional[MemorySummary]]:

									"""simple docstring"""
									lowercase										=							self._prepare_inference_func(a ,		a ,		a    )
									return self._measure_memory(_inference    )

							def 					_lowerCAmelCase  ( self       :    Union[str, Any] ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    float:

									"""simple docstring"""
									lowercase										=							self._prepare_train_func(a ,		a ,		a    )
									return self._measure_speed(_train    )

							def 					_lowerCAmelCase  ( self       :    int ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    [Memory, Optional[MemorySummary]]:

									"""simple docstring"""
									lowercase										=							self._prepare_train_func(a ,		a ,		a    )
									return self._measure_memory(_train    )

							def 					_lowerCAmelCase  ( self       :    List[Any] ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    Callable[[], None]:

									"""simple docstring"""
									lowercase										=							self.config_dict[model_name]

									if self.args.torchscript:
											lowercase										=							True

									lowercase										=							(
									    hasattr(a ,		'''architectures'''    )
									    and isinstance(config.architectures ,		a    )
									    and len(config.architectures    ) > 0
									)
									if not self.args.only_pretrain_model and has_model_class_in_config:
											try:
													lowercase										=							config.architectures[0]
													lowercase										=							__import__('''transformers''' ,		fromlist=[model_class]    )
													lowercase										=							getattr(a ,		a    )
													lowercase										=							model_cls(a    )
											except ImportError:
													raise ImportError(
													    f"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
													    ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'''    )
									else:
											lowercase										=							MODEL_MAPPING[config.__class__](a    )

									model.eval()
									model.to(self.args.device    )

									# encoder-decoder has vocab size saved differently
									lowercase										=							config.vocab_size if hasattr(a ,		'''vocab_size'''    ) else config.encoder.vocab_size
									lowercase										=							torch.randint(a ,		(batch_size, sequence_length) ,		dtype=torch.long ,		device=self.args.device    )

									if self.args.fpaa:
											logger.info('''Running training in Mixed Precision...'''    )
											if not self.args.is_gpu:
													raise ValueError('''Mixed precision is possible only for GPU.'''    )
											# amp seems to have memory leaks so that memory usage
											# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
											model.half()

									if self.args.torchscript:
											with torch.no_grad():
													lowercase										=							torch.jit.trace(a ,		a    )
									else:
											lowercase										=							model

									def encoder_decoder_forward():
											with torch.no_grad():
													lowercase										=							inference_model(a ,		decoder_input_ids=a    )
											return outputs

									def encoder_forward():
											with torch.no_grad():
													lowercase										=							inference_model(a    )
											return outputs

									lowercase										=							encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
									return _forward

							def 					_lowerCAmelCase  ( self       :    int ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    Callable[[], None]:

									"""simple docstring"""
									lowercase										=							self.config_dict[model_name]

									lowercase										=							(
									    hasattr(a ,		'''architectures'''    )
									    and isinstance(config.architectures ,		a    )
									    and len(config.architectures    ) > 0
									)
									if not self.args.only_pretrain_model and has_model_class_in_config:
											try:
													lowercase										=							config.architectures[0]
													lowercase										=							__import__('''transformers''' ,		fromlist=[model_class]    )
													lowercase										=							getattr(a ,		a    )
													lowercase										=							model_cls(a    )
											except ImportError:
													raise ImportError(
													    f"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
													    ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'''    )
									else:
											lowercase										=							MODEL_WITH_LM_HEAD_MAPPING[config.__class__](a    )

									if self.args.torchscript:
											raise NotImplementedError('''Training for torchscript is currently not implemented'''    )
									else:
											lowercase										=							model

									model.train()
									model.to(self.args.device    )

									# encoder-decoder has vocab size saved differently
									lowercase										=							config.vocab_size if hasattr(a ,		'''vocab_size'''    ) else config.encoder.vocab_size
									lowercase										=							torch.randint(a ,		(batch_size, sequence_length) ,		dtype=torch.long ,		device=self.args.device    )

									if self.args.fpaa:
											logger.info('''Running training in Mixed Precision...'''    )
											if not self.args.is_gpu:
													raise ValueError('''Mixed precision is possible only for GPU.'''    )

											# amp seems to have memory leaks so that memory usage
											# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
											model.half()

									def compute_loss_and_backprob_encoder():
											lowercase										=							train_model(a ,		labels=a    )[0]
											loss.backward()
											return loss

									def compute_loss_and_backprob_encoder_decoder():
											lowercase										=							train_model(a ,		decoder_input_ids=a ,		labels=a    )[0]
											loss.backward()
											return loss

									lowercase										=							(
									    compute_loss_and_backprob_encoder_decoder
									    if config.is_encoder_decoder
									    else compute_loss_and_backprob_encoder
									)
									return _train

							def 					_lowerCAmelCase  ( self       :    int ,		a       :    Optional[Any]    )   ->    float:

									"""simple docstring"""
									try:
											if self.args.is_tpu or self.args.torchscript:
													# run additional 10 times to stabilize compilation for tpu and torchscript
													logger.info('''Do inference on TPU or torchscript. Running model 5 times to stabilize compilation'''    )
													timeit.repeat(
													    a ,		repeat=1 ,		number=5 ,		)

											# as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
											lowercase										=							timeit.repeat(
											    a ,		repeat=self.args.repeat ,		number=10 ,		)

											if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
													import torch_xla.debug.metrics as met

													self.print_fn(met.metrics_report()    )

											return min(a    ) / 10.0
									except RuntimeError as e:
											self.print_fn(f"""Doesn't fit on GPU. {e}"""    )
											return "N/A"

							def 					_lowerCAmelCase  ( self       :    str ,		a       :    Callable[[], None]    )   ->    [Memory, MemorySummary]:

									"""simple docstring"""
									try:
											if self.args.trace_memory_line_by_line:
													lowercase										=							start_memory_tracing('''transformers'''    )

											if self.args.is_tpu:
													# tpu
													raise NotImplementedError(
													    '''Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with'''
													    ''' `--no-memory` or `args.memory=False`'''    )
											elif self.args.is_gpu:
													if not is_pyanvml_available():
															logger.warning(
															    '''py3nvml not installed, we won\'t log GPU memory usage. '''
															    '''Install py3nvml (pip install py3nvml) to log information about GPU.'''    )
															lowercase										=							'''N/A'''
													else:
															logger.info(
															    '''Measuring total GPU usage on GPU device. Make sure to not have additional processes running'''
															    ''' on the same GPU.'''    )
															# init nvml
															nvml.nvmlInit()
															func()
															lowercase										=							nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx    )
															lowercase										=							nvml.nvmlDeviceGetMemoryInfo(a    )
															lowercase										=							meminfo.used
															lowercase										=							Memory(a    )
															# shutdown nvml
															nvml.nvmlShutdown()
											else:
													# cpu
													lowercase										=							measure_peak_memory_cpu(a    )
													lowercase										=							Memory(a    ) if isinstance(a ,		a    ) else memory_bytes

											if self.args.trace_memory_line_by_line:
													lowercase										=							stop_memory_tracing(a    )
											else:
													lowercase										=							None

											return memory, summary
									except RuntimeError as e:
											self.print_fn(f"""Doesn't fit on GPU. {e}"""    )
											return "N/A", None