



'''simple docstring'''







import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
							import torch

							from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
							import pyanvml.pyanvml as nvml


SCREAMING_SNAKE_CASE__     : Optional[int]				=   logging.get_logger(__name__)




class 				__lowerCAmelCase(							lowerCAmelCase__      ):
			__snake_case  :		PyTorchBenchmarkArguments
			__snake_case  :		PretrainedConfig
			__snake_case  :		str		=      "PyTorch"






			@property
			def 				_lowercase	(   self  :							Dict	):




					"""simple docstring"""







					return torch.__version__






			def 				_lowercase	(   self  :							Dict      ,       SCREAMING_SNAKE_CASE  :							str      ,       SCREAMING_SNAKE_CASE  :							int      ,       SCREAMING_SNAKE_CASE  :							int	):




					"""simple docstring"""







					SCREAMING_SNAKE_CASE_						:Tuple										=      self._prepare_inference_func(SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE	)
					return self._measure_speed(_inference	)






			def 				_lowercase	(   self  :							int      ,       SCREAMING_SNAKE_CASE  :							str      ,       SCREAMING_SNAKE_CASE  :							int      ,       SCREAMING_SNAKE_CASE  :							int	):




					"""simple docstring"""







					SCREAMING_SNAKE_CASE_						:Any										=      self._prepare_inference_func(SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE	)
					return self._measure_memory(_inference	)






			def 				_lowercase	(   self  :							int      ,       SCREAMING_SNAKE_CASE  :							str      ,       SCREAMING_SNAKE_CASE  :							int      ,       SCREAMING_SNAKE_CASE  :							int	):




					"""simple docstring"""







					SCREAMING_SNAKE_CASE_						:Optional[int]										=      self._prepare_train_func(SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE	)
					return self._measure_speed(_train	)






			def 				_lowercase	(   self  :							List[Any]      ,       SCREAMING_SNAKE_CASE  :							str      ,       SCREAMING_SNAKE_CASE  :							int      ,       SCREAMING_SNAKE_CASE  :							int	):




					"""simple docstring"""







					SCREAMING_SNAKE_CASE_						:Tuple										=      self._prepare_train_func(SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE	)
					return self._measure_memory(_train	)






			def 				_lowercase	(   self  :							Optional[int]      ,       SCREAMING_SNAKE_CASE  :							str      ,       SCREAMING_SNAKE_CASE  :							int      ,       SCREAMING_SNAKE_CASE  :							int	):




					"""simple docstring"""







					SCREAMING_SNAKE_CASE_						:List[str]										=      self.config_dict[model_name]

					if self.args.torchscript:
							SCREAMING_SNAKE_CASE_						:List[str]										=      True

					SCREAMING_SNAKE_CASE_						:Any										=      (
					    hasattr(SCREAMING_SNAKE_CASE      ,       'architectures'	)
					    and isinstance(config.architectures      ,       SCREAMING_SNAKE_CASE	)
					    and len(config.architectures	) > 0
					)
					if not self.args.only_pretrain_model and has_model_class_in_config:
							try:
									SCREAMING_SNAKE_CASE_						:Optional[Any]										=      config.architectures[0]
									SCREAMING_SNAKE_CASE_						:List[Any]										=      __import__('transformers'      ,       fromlist=[model_class]	)
									SCREAMING_SNAKE_CASE_						:List[str]										=      getattr(SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE	)
									SCREAMING_SNAKE_CASE_						:Union[str, Any]										=      model_cls(SCREAMING_SNAKE_CASE	)
							except ImportError:
									raise ImportError(
									    f'{model_class} does not exist. If you just want to test the pretrained model, you might want to'
									    ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'	)
					else:
							SCREAMING_SNAKE_CASE_						:Tuple										=      MODEL_MAPPING[config.__class__](SCREAMING_SNAKE_CASE	)

					model.eval()
					model.to(self.args.device	)

					# encoder-decoder has vocab size saved differently
					SCREAMING_SNAKE_CASE_						:List[Any]										=      config.vocab_size if hasattr(SCREAMING_SNAKE_CASE      ,       'vocab_size'	) else config.encoder.vocab_size
					SCREAMING_SNAKE_CASE_						:Union[str, Any]										=      torch.randint(SCREAMING_SNAKE_CASE      ,       (batch_size, sequence_length)      ,       dtype=torch.long      ,       device=self.args.device	)

					if self.args.fpaa:
							logger.info('Running training in Mixed Precision...'	)
							if not self.args.is_gpu:
									raise ValueError('Mixed precision is possible only for GPU.'	)
							# amp seems to have memory leaks so that memory usage
							# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
							model.half()

					if self.args.torchscript:
							with torch.no_grad():
									SCREAMING_SNAKE_CASE_						:Optional[int]										=      torch.jit.trace(SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE	)
					else:
							SCREAMING_SNAKE_CASE_						:Optional[Any]										=      model

					def encoder_decoder_forward():
							with torch.no_grad():
									SCREAMING_SNAKE_CASE_						:Optional[Any]										=      inference_model(SCREAMING_SNAKE_CASE      ,       decoder_input_ids=SCREAMING_SNAKE_CASE	)
							return outputs

					def encoder_forward():
							with torch.no_grad():
									SCREAMING_SNAKE_CASE_						:str										=      inference_model(SCREAMING_SNAKE_CASE	)
							return outputs

					SCREAMING_SNAKE_CASE_						:Union[str, Any]										=      encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
					return _forward






			def 				_lowercase	(   self  :							Optional[int]      ,       SCREAMING_SNAKE_CASE  :							str      ,       SCREAMING_SNAKE_CASE  :							int      ,       SCREAMING_SNAKE_CASE  :							int	):




					"""simple docstring"""







					SCREAMING_SNAKE_CASE_						:int										=      self.config_dict[model_name]

					SCREAMING_SNAKE_CASE_						:int										=      (
					    hasattr(SCREAMING_SNAKE_CASE      ,       'architectures'	)
					    and isinstance(config.architectures      ,       SCREAMING_SNAKE_CASE	)
					    and len(config.architectures	) > 0
					)
					if not self.args.only_pretrain_model and has_model_class_in_config:
							try:
									SCREAMING_SNAKE_CASE_						:Union[str, Any]										=      config.architectures[0]
									SCREAMING_SNAKE_CASE_						:int										=      __import__('transformers'      ,       fromlist=[model_class]	)
									SCREAMING_SNAKE_CASE_						:Union[str, Any]										=      getattr(SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE	)
									SCREAMING_SNAKE_CASE_						:int										=      model_cls(SCREAMING_SNAKE_CASE	)
							except ImportError:
									raise ImportError(
									    f'{model_class} does not exist. If you just want to test the pretrained model, you might want to'
									    ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'	)
					else:
							SCREAMING_SNAKE_CASE_						:List[Any]										=      MODEL_WITH_LM_HEAD_MAPPING[config.__class__](SCREAMING_SNAKE_CASE	)

					if self.args.torchscript:
							raise NotImplementedError('Training for torchscript is currently not implemented'	)
					else:
							SCREAMING_SNAKE_CASE_						:List[Any]										=      model

					model.train()
					model.to(self.args.device	)

					# encoder-decoder has vocab size saved differently
					SCREAMING_SNAKE_CASE_						:List[Any]										=      config.vocab_size if hasattr(SCREAMING_SNAKE_CASE      ,       'vocab_size'	) else config.encoder.vocab_size
					SCREAMING_SNAKE_CASE_						:List[str]										=      torch.randint(SCREAMING_SNAKE_CASE      ,       (batch_size, sequence_length)      ,       dtype=torch.long      ,       device=self.args.device	)

					if self.args.fpaa:
							logger.info('Running training in Mixed Precision...'	)
							if not self.args.is_gpu:
									raise ValueError('Mixed precision is possible only for GPU.'	)

							# amp seems to have memory leaks so that memory usage
							# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
							model.half()

					def compute_loss_and_backprob_encoder():
							SCREAMING_SNAKE_CASE_						:str										=      train_model(SCREAMING_SNAKE_CASE      ,       labels=SCREAMING_SNAKE_CASE	)[0]
							loss.backward()
							return loss

					def compute_loss_and_backprob_encoder_decoder():
							SCREAMING_SNAKE_CASE_						:Optional[int]										=      train_model(SCREAMING_SNAKE_CASE      ,       decoder_input_ids=SCREAMING_SNAKE_CASE      ,       labels=SCREAMING_SNAKE_CASE	)[0]
							loss.backward()
							return loss

					SCREAMING_SNAKE_CASE_						:List[str]										=      (
					    compute_loss_and_backprob_encoder_decoder
					    if config.is_encoder_decoder
					    else compute_loss_and_backprob_encoder
					)
					return _train






			def 				_lowercase	(   self  :							List[Any]      ,       SCREAMING_SNAKE_CASE  :							int	):




					"""simple docstring"""







					try:
							if self.args.is_tpu or self.args.torchscript:
									# run additional 10 times to stabilize compilation for tpu and torchscript
									logger.info('Do inference on TPU or torchscript. Running model 5 times to stabilize compilation'	)
									timeit.repeat(
									    SCREAMING_SNAKE_CASE      ,       repeat=1      ,       number=5      ,       )

							# as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
							SCREAMING_SNAKE_CASE_						:List[Any]										=      timeit.repeat(
							    SCREAMING_SNAKE_CASE      ,       repeat=self.args.repeat      ,       number=10      ,       )

							if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
									import torch_xla.debug.metrics as met

									self.print_fn(met.metrics_report()	)

							return min(SCREAMING_SNAKE_CASE	) / 10.0
					except RuntimeError as e:
							self.print_fn(f'Doesn\'t fit on GPU. {e}'	)
							return "N/A"






			def 				_lowercase	(   self  :							Optional[Any]      ,       SCREAMING_SNAKE_CASE  :							Callable[[], None]	):




					"""simple docstring"""







					try:
							if self.args.trace_memory_line_by_line:
									SCREAMING_SNAKE_CASE_						:Tuple										=      start_memory_tracing('transformers'	)

							if self.args.is_tpu:
									# tpu
									raise NotImplementedError(
									    'Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with'
									    ' `--no-memory` or `args.memory=False`'	)
							elif self.args.is_gpu:
									if not is_pyanvml_available():
											logger.warning(
											    'py3nvml not installed, we won\'t log GPU memory usage. '
											    'Install py3nvml (pip install py3nvml) to log information about GPU.'	)
											SCREAMING_SNAKE_CASE_						:Tuple										=      'N/A'
									else:
											logger.info(
											    'Measuring total GPU usage on GPU device. Make sure to not have additional processes running'
											    ' on the same GPU.'	)
											# init nvml
											nvml.nvmlInit()
											func()
											SCREAMING_SNAKE_CASE_						:Optional[int]										=      nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx	)
											SCREAMING_SNAKE_CASE_						:Any										=      nvml.nvmlDeviceGetMemoryInfo(SCREAMING_SNAKE_CASE	)
											SCREAMING_SNAKE_CASE_						:Dict										=      meminfo.used
											SCREAMING_SNAKE_CASE_						:int										=      Memory(SCREAMING_SNAKE_CASE	)
											# shutdown nvml
											nvml.nvmlShutdown()
							else:
									# cpu
									SCREAMING_SNAKE_CASE_						:Optional[int]										=      measure_peak_memory_cpu(SCREAMING_SNAKE_CASE	)
									SCREAMING_SNAKE_CASE_						:Any										=      Memory(SCREAMING_SNAKE_CASE	) if isinstance(SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE	) else memory_bytes

							if self.args.trace_memory_line_by_line:
									SCREAMING_SNAKE_CASE_						:Optional[Any]										=      stop_memory_tracing(SCREAMING_SNAKE_CASE	)
							else:
									SCREAMING_SNAKE_CASE_						:int										=      None

							return memory, summary
					except RuntimeError as e:
							self.print_fn(f'Doesn\'t fit on GPU. {e}'	)
							return "N/A", None





'''simple docstring'''







def  SCREAMING_SNAKE_CASE__ (				SCREAMING_SNAKE_CASE = 1000     ):
		return sum(i for i in range(SCREAMING_SNAKE_CASE     ) if i % 3 == 0 or i % 5 == 0     )


if __name__ == "__main__":
							print(f"""{solution() = }""")
