





import random
import timeit
from functools import wraps
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_tf_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_tf_available():
    import tensorflow as tf
    from tensorflow.python.framework.errors_impl import ResourceExhaustedError

    from .benchmark_args_tf import TensorFlowBenchmarkArguments

if is_pyanvml_available():
    import pyanvml.pyanvml as nvml

__lowerCamelCase       :     Any   =     logging.get_logger(__name__)



def 		A_				(     _lowerCAmelCase							,   _lowerCAmelCase     )						-> List[Any]:
 def run_func(_lowerCAmelCase     ):
  @wraps(_lowerCAmelCase     )
  def run_in_eager_mode(*_lowerCAmelCase							,   **_lowerCAmelCase     ):
   return func(*_lowerCAmelCase							,   **_lowerCAmelCase     )

  @wraps(_lowerCAmelCase     )
  @tf.function(experimental_compile=_lowerCAmelCase     )
  def run_in_graph_mode(*_lowerCAmelCase							,   **_lowerCAmelCase     ):
   return func(*_lowerCAmelCase							,   **_lowerCAmelCase     )

  if do_eager_mode is True:
   if use_xla is not False:
    raise ValueError(
        "Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`."     )
   return run_in_eager_mode
  else:
   return run_in_graph_mode

 return run_func



def 		A_				(     _lowerCAmelCase							,   _lowerCAmelCase							,   _lowerCAmelCase     )						-> ["tf.Tensor"]:
 UpperCamelCase	:					Optional[Any]       	=   random.Random()
 UpperCamelCase	:					str       	=   [rng.randint(0							,   vocab_size - 1     ) for i in range(batch_size * sequence_length     )]
 return tf.constant(_lowerCAmelCase							,   shape=(batch_size, sequence_length)							,   dtype=tf.intaa     )



class 					A__   (       __snake_case			):
      _UpperCAmelCase	:TensorFlowBenchmarkArguments
      _UpperCAmelCase	:PretrainedConfig
      _UpperCAmelCase	:str						    =   "TensorFlow"





      @property
      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       return tf.__version__





      def        __UpperCamelCase( self       ,							A_       ,							A_       ,							A_   ):
       '''simple docstring'''




       UpperCamelCase	:					Optional[int]       	=   self.args.strategy
       if strategy is None:
        raise ValueError("A device strategy has to be initialized before using TensorFlow."   )
       UpperCamelCase	:					Optional[Any]       	=   self._prepare_inference_func(A_       ,							A_       ,							A_   )
       return self._measure_speed(_inference   )





      def        __UpperCamelCase( self       ,							A_       ,							A_       ,							A_   ):
       '''simple docstring'''




       UpperCamelCase	:					Any       	=   self.args.strategy
       if strategy is None:
        raise ValueError("A device strategy has to be initialized before using TensorFlow."   )
       UpperCamelCase	:					Any       	=   self._prepare_train_func(A_       ,							A_       ,							A_   )
       return self._measure_speed(_train   )





      def        __UpperCamelCase( self       ,							A_       ,							A_       ,							A_   ):
       '''simple docstring'''




       if self.args.is_gpu:
        tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]       ,							A_   )
       UpperCamelCase	:					Union[str, Any]       	=   self.args.strategy
       if strategy is None:
        raise ValueError("A device strategy has to be initialized before using TensorFlow."   )
       UpperCamelCase	:					Union[str, Any]       	=   self._prepare_inference_func(A_       ,							A_       ,							A_   )
       return self._measure_memory(_inference   )





      def        __UpperCamelCase( self       ,							A_       ,							A_       ,							A_   ):
       '''simple docstring'''




       if self.args.is_gpu:
        tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]       ,							A_   )
       UpperCamelCase	:					Union[str, Any]       	=   self.args.strategy
       if strategy is None:
        raise ValueError("A device strategy has to be initialized before using TensorFlow."   )

       UpperCamelCase	:					Union[str, Any]       	=   self._prepare_train_func(A_       ,							A_       ,							A_   )
       return self._measure_memory(_train   )





      def        __UpperCamelCase( self       ,							A_       ,							A_       ,							A_   ):
       '''simple docstring'''




       UpperCamelCase	:					Tuple       	=   self.config_dict[model_name]

       if self.args.fpaa:
        raise NotImplementedError("Mixed precision is currently not supported."   )

       UpperCamelCase	:					Union[str, Any]       	=   (
           hasattr(A_       ,							"architectures"   )
           and isinstance(config.architectures       ,							A_   )
           and len(config.architectures   ) > 0
       )
       if not self.args.only_pretrain_model and has_model_class_in_config:
        try:
         UpperCamelCase	:					Optional[Any]       	=   "TF" + config.architectures[0]  # prepend 'TF' for tensorflow model
         UpperCamelCase	:					str       	=   __import__("transformers"       ,							fromlist=[model_class]   )
         UpperCamelCase	:					Dict       	=   getattr(A_       ,							A_   )
         UpperCamelCase	:					List[str]       	=   model_cls(A_   )
        except ImportError:
         raise ImportError(
             F"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
             " set `--only_pretrain_model` or `args.only_pretrain_model=True`."   )
       else:
        UpperCamelCase	:					Tuple       	=   TF_MODEL_MAPPING[config.__class__](A_   )

       # encoder-decoder has vocab size saved differently
       UpperCamelCase	:					Optional[int]       	=   config.vocab_size if hasattr(A_       ,							"vocab_size"   ) else config.encoder.vocab_size
       UpperCamelCase	:					str       	=   random_input_ids(A_       ,							A_       ,							A_   )

       @run_with_tf_optimizations(self.args.eager_mode       ,							self.args.use_xla   )
       def encoder_decoder_forward():
        return model(A_       ,							decoder_input_ids=A_       ,							training=A_   )

       @run_with_tf_optimizations(self.args.eager_mode       ,							self.args.use_xla   )
       def encoder_forward():
        return model(A_       ,							training=A_   )

       UpperCamelCase	:					Dict       	=   encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

       return _inference





      def        __UpperCamelCase( self       ,							A_       ,							A_       ,							A_   ):
       '''simple docstring'''




       UpperCamelCase	:					Union[str, Any]       	=   self.config_dict[model_name]

       if self.args.eager_mode is not False:
        raise ValueError("Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`."   )

       if self.args.fpaa:
        raise NotImplementedError("Mixed precision is currently not supported."   )

       UpperCamelCase	:					List[Any]       	=   (
           hasattr(A_       ,							"architectures"   )
           and isinstance(config.architectures       ,							A_   )
           and len(config.architectures   ) > 0
       )
       if not self.args.only_pretrain_model and has_model_class_in_config:
        try:
         UpperCamelCase	:					Any       	=   "TF" + config.architectures[0]  # prepend 'TF' for tensorflow model
         UpperCamelCase	:					Dict       	=   __import__("transformers"       ,							fromlist=[model_class]   )
         UpperCamelCase	:					Any       	=   getattr(A_       ,							A_   )
         UpperCamelCase	:					List[str]       	=   model_cls(A_   )
        except ImportError:
         raise ImportError(
             F"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
             " set `--only_pretrain_model` or `args.only_pretrain_model=True`."   )
       else:
        UpperCamelCase	:					int       	=   TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](A_   )

       # encoder-decoder has vocab size saved differently
       UpperCamelCase	:					Optional[Any]       	=   config.vocab_size if hasattr(A_       ,							"vocab_size"   ) else config.encoder.vocab_size
       UpperCamelCase	:					Any       	=   random_input_ids(A_       ,							A_       ,							A_   )

       @run_with_tf_optimizations(self.args.eager_mode       ,							self.args.use_xla   )
       def encoder_decoder_train():
        UpperCamelCase	:					List[Any]       	=   model(A_       ,							decoder_input_ids=A_       ,							labels=A_       ,							training=A_   )[0]
        UpperCamelCase	:					Dict       	=   tf.gradients(A_       ,							model.trainable_variables   )
        return gradients

       @run_with_tf_optimizations(self.args.eager_mode       ,							self.args.use_xla   )
       def encoder_train():
        UpperCamelCase	:					Union[str, Any]       	=   model(A_       ,							labels=A_       ,							training=A_   )[0]
        UpperCamelCase	:					Optional[int]       	=   tf.gradients(A_       ,							model.trainable_variables   )
        return gradients

       UpperCamelCase	:					Union[str, Any]       	=   encoder_decoder_train if config.is_encoder_decoder else encoder_train

       return _train





      def        __UpperCamelCase( self       ,							A_   ):
       '''simple docstring'''




       with self.args.strategy.scope():
        try:
         if self.args.is_tpu or self.args.use_xla:
          # run additional 10 times to stabilize compilation for tpu
          logger.info("Do inference on TPU. Running model 5 times to stabilize compilation"   )
          timeit.repeat(A_       ,							repeat=1       ,							number=5   )

         # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
         UpperCamelCase	:					str       	=   timeit.repeat(
             A_       ,							repeat=self.args.repeat       ,							number=10       ,							)

         return min(A_   ) / 10.0
        except ResourceExhaustedError as e:
         self.print_fn(F"""Doesn't fit on GPU. {e}"""   )





      def        __UpperCamelCase( self       ,							A_   ):
       '''simple docstring'''




       logger.info(
           "Note that TensorFlow allocates more memory than "
           "it might need to speed up computation. "
           "The memory reported here corresponds to the memory "
           "reported by `nvidia-smi`, which can vary depending "
           "on total available memory on the GPU that is used."   )
       with self.args.strategy.scope():
        try:
         if self.args.trace_memory_line_by_line:
          if not self.args.eager_mode:
           raise ValueError(
               "`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory"
               " consumption line by line."   )
          UpperCamelCase	:					Dict       	=   start_memory_tracing("transformers"   )

         if self.args.is_tpu:
          # tpu
          raise NotImplementedError(
              "Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking"
              " with `args.memory=False`"   )
         elif self.args.is_gpu:
          # gpu
          if not is_pyanvml_available():
           logger.warning(
               "py3nvml not installed, we won't log GPU memory usage. "
               "Install py3nvml (pip install py3nvml) to log information about GPU."   )
           UpperCamelCase	:					Any       	=   "N/A"
          else:
           logger.info(
               "Measuring total GPU usage on GPU device. Make sure to not have additional processes"
               " running on the same GPU."   )
           # init nvml
           nvml.nvmlInit()
           func()
           UpperCamelCase	:					List[Any]       	=   nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx   )
           UpperCamelCase	:					int       	=   nvml.nvmlDeviceGetMemoryInfo(A_   )
           UpperCamelCase	:					Tuple       	=   meminfo.used
           UpperCamelCase	:					List[str]       	=   Memory(A_   )
           # shutdown nvml
           nvml.nvmlShutdown()
         else:
          # cpu
          if self.args.trace_memory_line_by_line:
           logger.info(
               "When enabling line by line tracing, the max peak memory for CPU is inaccurate in"
               " TensorFlow."   )
           UpperCamelCase	:					Optional[Any]       	=   None
          else:
           UpperCamelCase	:					List[str]       	=   measure_peak_memory_cpu(A_   )
           UpperCamelCase	:					Dict       	=   Memory(A_   ) if isinstance(A_       ,							A_   ) else memory_bytes
         if self.args.trace_memory_line_by_line:
          UpperCamelCase	:					int       	=   stop_memory_tracing(A_   )
          if memory is None:
           UpperCamelCase	:					str       	=   summary.total
         else:
          UpperCamelCase	:					List[Any]       	=   None

         return memory, summary
        except ResourceExhaustedError as e:
         self.print_fn(F"""Doesn't fit on GPU. {e}"""   )
         return "N/A", None







class 					A__   :





      def __init__( self       ,							A_   ):
       '''simple docstring'''




       UpperCamelCase	:					Union[str, Any]       	=   set_counts
       UpperCamelCase	:					int       	=   max(A_   )
       UpperCamelCase	:					Optional[Any]       	=   len(A_   )
       UpperCamelCase	:					Union[str, Any]       	=   [1] * num_sets
       UpperCamelCase	:					Union[str, Any]       	=   list(range(A_   )   )





      def        __UpperCamelCase( self       ,							A_       ,							A_   ):
       '''simple docstring'''




       UpperCamelCase	:					Any       	=   self.get_parent(A_   )
       UpperCamelCase	:					Optional[int]       	=   self.get_parent(A_   )

       if src_parent == dst_parent:
        return False

       if self.ranks[dst_parent] >= self.ranks[src_parent]:
        self.set_counts[dst_parent] += self.set_counts[src_parent]
        UpperCamelCase	:					int       	=   0
        UpperCamelCase	:					Dict       	=   dst_parent
        if self.ranks[dst_parent] == self.ranks[src_parent]:
         self.ranks[dst_parent] += 1
        UpperCamelCase	:					Optional[int]       	=   self.set_counts[dst_parent]
       else:
        self.set_counts[src_parent] += self.set_counts[dst_parent]
        UpperCamelCase	:					Any       	=   0
        UpperCamelCase	:					Optional[int]       	=   src_parent
        UpperCamelCase	:					int       	=   self.set_counts[src_parent]

       UpperCamelCase	:					Any       	=   max(self.max_set       ,							A_   )
       return True





      def        __UpperCamelCase( self       ,							A_   ):
       '''simple docstring'''




       if self.parents[disj_set] == disj_set:
        return disj_set
       UpperCamelCase	:					Optional[int]       	=   self.get_parent(self.parents[disj_set]   )
       return self.parents[disj_set]
