


'''simple docstring'''




import random
import timeit
from functools import wraps
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_tf_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_tf_available():
       import tensorflow as tf
       from tensorflow.python.framework.errors_impl import ResourceExhaustedError

       from .benchmark_args_tf import TensorFlowBenchmarkArguments

if is_pyanvml_available():
       import pyanvml.pyanvml as nvml

__snake_case    						=logging.get_logger(__name__)





def 	a_					( lowerCamelCase	:     bool	, lowerCamelCase	:     bool				):
   def run_func(lowerCamelCase	:     List[str]				):
      @wraps(lowerCamelCase				)
      def run_in_eager_mode(*lowerCamelCase	:     List[str]	, **lowerCamelCase	:     Tuple				):
         return func(*lowerCamelCase	, **lowerCamelCase				)

      @wraps(lowerCamelCase				)
      @tf.function(experimental_compile=lowerCamelCase				)
      def run_in_graph_mode(*lowerCamelCase	:     List[Any]	, **lowerCamelCase	:     Any				):
         return func(*lowerCamelCase	, **lowerCamelCase				)

      if do_eager_mode is True:
         if use_xla is not False:
            raise ValueError(
                'Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`.'				)
         return run_in_eager_mode
      else:
         return run_in_graph_mode

   return run_func





def 	a_					( lowerCamelCase	:     int	, lowerCamelCase	:     int	, lowerCamelCase	:     int				):
   lowerCAmelCase     =			random.Random()
   lowerCAmelCase     =			[rng.randint(0	, vocab_size - 1				) for i in range(batch_size * sequence_length				)]
   return tf.constant(lowerCamelCase	, shape=(batch_size, sequence_length)	, dtype=tf.intaa				)




class 		UpperCAmelCase_      (  __lowercase		):
     lowerCamelCase				:	TensorFlowBenchmarkArguments
     lowerCamelCase				:	PretrainedConfig
     lowerCamelCase				:	str		=     "TensorFlow"





     @property
     def 							__UpperCAmelCase      ( self       :      int    )				->  Optional[int]:
        return tf.__version__





     def 							__UpperCAmelCase      ( self       :      Any    ,      UpperCAmelCase__       :      str    ,      UpperCAmelCase__       :      int    ,      UpperCAmelCase__       :      int    )				->  float:
        # initialize GPU on separate process
        lowerCAmelCase     =			self.args.strategy
        if strategy is None:
           raise ValueError('A device strategy has to be initialized before using TensorFlow.'    )
        lowerCAmelCase     =			self._prepare_inference_func(UpperCAmelCase__    ,      UpperCAmelCase__    ,      UpperCAmelCase__    )
        return self._measure_speed(_inference    )





     def 							__UpperCAmelCase      ( self       :      Union[str, Any]    ,      UpperCAmelCase__       :      str    ,      UpperCAmelCase__       :      int    ,      UpperCAmelCase__       :      int    )				->  float:
        lowerCAmelCase     =			self.args.strategy
        if strategy is None:
           raise ValueError('A device strategy has to be initialized before using TensorFlow.'    )
        lowerCAmelCase     =			self._prepare_train_func(UpperCAmelCase__    ,      UpperCAmelCase__    ,      UpperCAmelCase__    )
        return self._measure_speed(_train    )





     def 							__UpperCAmelCase      ( self       :      Optional[int]    ,      UpperCAmelCase__       :      str    ,      UpperCAmelCase__       :      int    ,      UpperCAmelCase__       :      int    )				->  [Memory, Optional[MemorySummary]]:
        # initialize GPU on separate process
        if self.args.is_gpu:
           tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]    ,      UpperCAmelCase__    )
        lowerCAmelCase     =			self.args.strategy
        if strategy is None:
           raise ValueError('A device strategy has to be initialized before using TensorFlow.'    )
        lowerCAmelCase     =			self._prepare_inference_func(UpperCAmelCase__    ,      UpperCAmelCase__    ,      UpperCAmelCase__    )
        return self._measure_memory(_inference    )





     def 							__UpperCAmelCase      ( self       :      str    ,      UpperCAmelCase__       :      str    ,      UpperCAmelCase__       :      int    ,      UpperCAmelCase__       :      int    )				->  [Memory, Optional[MemorySummary]]:
        if self.args.is_gpu:
           tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]    ,      UpperCAmelCase__    )
        lowerCAmelCase     =			self.args.strategy
        if strategy is None:
           raise ValueError('A device strategy has to be initialized before using TensorFlow.'    )

        lowerCAmelCase     =			self._prepare_train_func(UpperCAmelCase__    ,      UpperCAmelCase__    ,      UpperCAmelCase__    )
        return self._measure_memory(_train    )





     def 							__UpperCAmelCase      ( self       :      Optional[int]    ,      UpperCAmelCase__       :      str    ,      UpperCAmelCase__       :      int    ,      UpperCAmelCase__       :      int    )				->  Callable[[], None]:
        lowerCAmelCase     =			self.config_dict[model_name]

        if self.args.fpaa:
           raise NotImplementedError('Mixed precision is currently not supported.'    )

        lowerCAmelCase     =			(
            hasattr(UpperCAmelCase__    ,      'architectures'    )
            and isinstance(config.architectures    ,      UpperCAmelCase__    )
            and len(config.architectures    ) > 0
        )
        if not self.args.only_pretrain_model and has_model_class_in_config:
           try:
              lowerCAmelCase     =			'TF' + config.architectures[0]  # prepend 'TF' for tensorflow model
              lowerCAmelCase     =			__import__('transformers'    ,      fromlist=[model_class]    )
              lowerCAmelCase     =			getattr(UpperCAmelCase__    ,      UpperCAmelCase__    )
              lowerCAmelCase     =			model_cls(UpperCAmelCase__    )
           except ImportError:
              raise ImportError(
                  F'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
                  ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'    )
        else:
           lowerCAmelCase     =			TF_MODEL_MAPPING[config.__class__](UpperCAmelCase__    )

        # encoder-decoder has vocab size saved differently
        lowerCAmelCase     =			config.vocab_size if hasattr(UpperCAmelCase__    ,      'vocab_size'    ) else config.encoder.vocab_size
        lowerCAmelCase     =			random_input_ids(UpperCAmelCase__    ,      UpperCAmelCase__    ,      UpperCAmelCase__    )

        @run_with_tf_optimizations(self.args.eager_mode    ,      self.args.use_xla    )
        def encoder_decoder_forward():
           return model(UpperCAmelCase__    ,      decoder_input_ids=UpperCAmelCase__    ,      training=UpperCAmelCase__    )

        @run_with_tf_optimizations(self.args.eager_mode    ,      self.args.use_xla    )
        def encoder_forward():
           return model(UpperCAmelCase__    ,      training=UpperCAmelCase__    )

        lowerCAmelCase     =			encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

        return _inference





     def 							__UpperCAmelCase      ( self       :      List[Any]    ,      UpperCAmelCase__       :      str    ,      UpperCAmelCase__       :      int    ,      UpperCAmelCase__       :      int    )				->  Callable[[], None]:
        lowerCAmelCase     =			self.config_dict[model_name]

        if self.args.eager_mode is not False:
           raise ValueError('Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`.'    )

        if self.args.fpaa:
           raise NotImplementedError('Mixed precision is currently not supported.'    )

        lowerCAmelCase     =			(
            hasattr(UpperCAmelCase__    ,      'architectures'    )
            and isinstance(config.architectures    ,      UpperCAmelCase__    )
            and len(config.architectures    ) > 0
        )
        if not self.args.only_pretrain_model and has_model_class_in_config:
           try:
              lowerCAmelCase     =			'TF' + config.architectures[0]  # prepend 'TF' for tensorflow model
              lowerCAmelCase     =			__import__('transformers'    ,      fromlist=[model_class]    )
              lowerCAmelCase     =			getattr(UpperCAmelCase__    ,      UpperCAmelCase__    )
              lowerCAmelCase     =			model_cls(UpperCAmelCase__    )
           except ImportError:
              raise ImportError(
                  F'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
                  ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'    )
        else:
           lowerCAmelCase     =			TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](UpperCAmelCase__    )

        # encoder-decoder has vocab size saved differently
        lowerCAmelCase     =			config.vocab_size if hasattr(UpperCAmelCase__    ,      'vocab_size'    ) else config.encoder.vocab_size
        lowerCAmelCase     =			random_input_ids(UpperCAmelCase__    ,      UpperCAmelCase__    ,      UpperCAmelCase__    )

        @run_with_tf_optimizations(self.args.eager_mode    ,      self.args.use_xla    )
        def encoder_decoder_train():
           lowerCAmelCase     =			model(UpperCAmelCase__    ,      decoder_input_ids=UpperCAmelCase__    ,      labels=UpperCAmelCase__    ,      training=UpperCAmelCase__    )[0]
           lowerCAmelCase     =			tf.gradients(UpperCAmelCase__    ,      model.trainable_variables    )
           return gradients

        @run_with_tf_optimizations(self.args.eager_mode    ,      self.args.use_xla    )
        def encoder_train():
           lowerCAmelCase     =			model(UpperCAmelCase__    ,      labels=UpperCAmelCase__    ,      training=UpperCAmelCase__    )[0]
           lowerCAmelCase     =			tf.gradients(UpperCAmelCase__    ,      model.trainable_variables    )
           return gradients

        lowerCAmelCase     =			encoder_decoder_train if config.is_encoder_decoder else encoder_train

        return _train





     def 							__UpperCAmelCase      ( self       :      Optional[int]    ,      UpperCAmelCase__       :      Optional[Any]    )				->  float:
        with self.args.strategy.scope():
           try:
              if self.args.is_tpu or self.args.use_xla:
                 # run additional 10 times to stabilize compilation for tpu
                 logger.info('Do inference on TPU. Running model 5 times to stabilize compilation'    )
                 timeit.repeat(UpperCAmelCase__    ,      repeat=1    ,      number=5    )

              # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
              lowerCAmelCase     =			timeit.repeat(
                  UpperCAmelCase__    ,      repeat=self.args.repeat    ,      number=1_0    ,      )

              return min(UpperCAmelCase__    ) / 10.0
           except ResourceExhaustedError as e:
              self.print_fn(F'''Doesn\'t fit on GPU. {e}'''    )





     def 							__UpperCAmelCase      ( self       :      Union[str, Any]    ,      UpperCAmelCase__       :      Callable[[], None]    )				->  [Memory, MemorySummary]:
        logger.info(
            'Note that TensorFlow allocates more memory than '
            'it might need to speed up computation. '
            'The memory reported here corresponds to the memory '
            'reported by `nvidia-smi`, which can vary depending '
            'on total available memory on the GPU that is used.'    )
        with self.args.strategy.scope():
           try:
              if self.args.trace_memory_line_by_line:
                 if not self.args.eager_mode:
                    raise ValueError(
                        '`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory'
                        ' consumption line by line.'    )
                 lowerCAmelCase     =			start_memory_tracing('transformers'    )

              if self.args.is_tpu:
                 # tpu
                 raise NotImplementedError(
                     'Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking'
                     ' with `args.memory=False`'    )
              elif self.args.is_gpu:
                 # gpu
                 if not is_pyanvml_available():
                    logger.warning(
                        'py3nvml not installed, we won\'t log GPU memory usage. '
                        'Install py3nvml (pip install py3nvml) to log information about GPU.'    )
                    lowerCAmelCase     =			'N/A'
                 else:
                    logger.info(
                        'Measuring total GPU usage on GPU device. Make sure to not have additional processes'
                        ' running on the same GPU.'    )
                    # init nvml
                    nvml.nvmlInit()
                    func()
                    lowerCAmelCase     =			nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx    )
                    lowerCAmelCase     =			nvml.nvmlDeviceGetMemoryInfo(UpperCAmelCase__    )
                    lowerCAmelCase     =			meminfo.used
                    lowerCAmelCase     =			Memory(UpperCAmelCase__    )
                    # shutdown nvml
                    nvml.nvmlShutdown()
              else:
                 # cpu
                 if self.args.trace_memory_line_by_line:
                    logger.info(
                        'When enabling line by line tracing, the max peak memory for CPU is inaccurate in'
                        ' TensorFlow.'    )
                    lowerCAmelCase     =			None
                 else:
                    lowerCAmelCase     =			measure_peak_memory_cpu(UpperCAmelCase__    )
                    lowerCAmelCase     =			Memory(UpperCAmelCase__    ) if isinstance(UpperCAmelCase__    ,      UpperCAmelCase__    ) else memory_bytes
              if self.args.trace_memory_line_by_line:
                 lowerCAmelCase     =			stop_memory_tracing(UpperCAmelCase__    )
                 if memory is None:
                    lowerCAmelCase     =			summary.total
              else:
                 lowerCAmelCase     =			None

              return memory, summary
           except ResourceExhaustedError as e:
              self.print_fn(F'''Doesn\'t fit on GPU. {e}'''    )
              return "N/A", None









'''simple docstring'''




import logging
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional, Union

from .generation.configuration_utils import GenerationConfig
from .training_args import TrainingArguments
from .utils import add_start_docstrings


__snake_case    						=logging.getLogger(__name__)

@dataclass
@add_start_docstrings(TrainingArguments.__doc__		)
class 		UpperCAmelCase_      (  __lowercase		):
     lowerCamelCase				:	bool		=     field(default=__lowercase       ,	metadata={'''help''': '''Whether to use SortishSampler or not.'''}		)
     lowerCamelCase				:	bool		=     field(
         default=__lowercase       ,	metadata={'''help''': '''Whether to use generate to calculate generative metrics (ROUGE, BLEU).'''}		)
     lowerCamelCase				:	Optional[int]		=     field(
         default=__lowercase       ,	metadata={
             '''help''': (
                 '''The `max_length` to use on each evaluation loop when `predict_with_generate=True`. Will default '''
                 '''to the `max_length` value of the model configuration.'''
             )
         }       ,	)
     lowerCamelCase				:	Optional[int]		=     field(
         default=__lowercase       ,	metadata={
             '''help''': (
                 '''The `num_beams` to use on each evaluation loop when `predict_with_generate=True`. Will default '''
                 '''to the `num_beams` value of the model configuration.'''
             )
         }       ,	)
     lowerCamelCase				:	Optional[Union[str, Path, GenerationConfig]]		=     field(
         default=__lowercase       ,	metadata={
             '''help''': '''Model id, file path or url pointing to a GenerationConfig json file, to use during prediction.'''
         }       ,	)





     def 							__UpperCAmelCase      ( self       :      Dict    )				->  List[str]:
        lowerCAmelCase     =			super().to_dict()
        for k, v in d.items():
           if isinstance(UpperCAmelCase__    ,      UpperCAmelCase__    ):
              lowerCAmelCase     =			v.to_dict()
        return d





