




# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import subprocess

from packaging.version import Version, parse

from accelerate.commands.config.config_args import default_config_file, load_config_from_file


snake_case_ : Optional[int]    		=       "Run commands across TPU VMs for initial setup before running `accelerate launch`."


def    A						(__A   :    str=None       )   ->      Optional[int]:







     """simple docstring"""


     if subparsers is not None:
          UpperCAmelCase_          =							subparsers.add_parser('''tpu-config'''   ,				description=_description       )
     else:
          UpperCAmelCase_          =							argparse.ArgumentParser('''Accelerate tpu-config command'''   ,				description=_description       )
     # Core arguments
     UpperCAmelCase_          =							parser.add_argument_group(
         '''Config Arguments'''   ,				'''Arguments that can be configured through `accelerate config`.'''       )
     config_args.add_argument(
         '''--config_file'''   ,				type=__A   ,				default=__A   ,				help='''Path to the config file to use for accelerate.'''   ,				)
     config_args.add_argument(
         '''--tpu_name'''   ,				default=__A   ,				help='''The name of the TPU to use. If not specified, will use the TPU specified in the config file.'''   ,				)
     config_args.add_argument(
         '''--tpu_zone'''   ,				default=__A   ,				help='''The zone of the TPU to use. If not specified, will use the zone specified in the config file.'''   ,				)
     UpperCAmelCase_          =							parser.add_argument_group('''TPU Arguments'''   ,				'''Arguments for options ran inside the TPU.'''       )
     pod_args.add_argument(
         '''--use_alpha'''   ,				action='''store_true'''   ,				help='''Whether to use `gcloud alpha` when running the TPU training script instead of `gcloud`.'''   ,				)
     pod_args.add_argument(
         '''--command_file'''   ,				default=__A   ,				help='''The path to the file containing the commands to run on the pod on startup.'''   ,				)
     pod_args.add_argument(
         '''--command'''   ,				action='''append'''   ,				nargs='''+'''   ,				help='''A command to run on the pod. Can be passed multiple times.'''   ,				)
     pod_args.add_argument(
         '''--install_accelerate'''   ,				action='''store_true'''   ,				help='''Whether to install accelerate on the pod. Defaults to False.'''   ,				)
     pod_args.add_argument(
         '''--accelerate_version'''   ,				default='''latest'''   ,				help='''The version of accelerate to install on the pod. If not specified, will use the latest pypi version. Specify \'dev\' to install from GitHub.'''   ,				)
     pod_args.add_argument(
         '''--debug'''   ,				action='''store_true'''   ,				help='''If set, will print the command that would be run instead of running it.'''       )

     if subparsers is not None:
          parser.set_defaults(func=__A       )
     return parser


def    A						(__A   :    str       )   ->      Union[str, Any]:







     """simple docstring"""


     UpperCAmelCase_          =							None

     # Get the default from the config file if it exists.
     if args.config_file is not None or os.path.isfile(__A       ):
          UpperCAmelCase_          =							load_config_from_file(args.config_file       )
          if not args.command_file and defaults.command_file is not None and not args.command:
               UpperCAmelCase_          =							defaults.command_file
          if not args.command and defaults.commands is not None:
               UpperCAmelCase_          =							defaults.commands
          if not args.tpu_name:
               UpperCAmelCase_          =							defaults.tpu_name
          if not args.tpu_zone:
               UpperCAmelCase_          =							defaults.tpu_zone
     if args.accelerate_version == "dev":
          UpperCAmelCase_          =							'''git+https://github.com/huggingface/accelerate.git'''
     elif args.accelerate_version == "latest":
          UpperCAmelCase_          =							'''accelerate -U'''
     elif isinstance(parse(args.accelerate_version       )   ,				__A       ):
          UpperCAmelCase_          =							F"""accelerate=={args.accelerate_version}"""

     if not args.command_file and not args.command:
          raise ValueError('''You must specify either a command file or a command to run on the pod.'''       )

     if args.command_file:
          with open(args.command_file   ,				'''r'''       ) as f:
               UpperCAmelCase_          =							[f.read().splitlines()]

    # To turn list of lists into list of strings
     if isinstance(args.command[0]   ,				__A       ):
          UpperCAmelCase_          =							[line for cmd in args.command for line in cmd]
     # Default to the shared folder and install accelerate
     UpperCAmelCase_          =							['''cd /usr/share''']
     if args.install_accelerate:
          new_cmd += [F"""pip install {args.accelerate_version}"""]
     new_cmd += args.command
     UpperCAmelCase_          =							'''; '''.join(__A       )

     # Then send it to gcloud
     # Eventually try to use google-api-core to do this instead of subprocess
     UpperCAmelCase_          =							['''gcloud''']
     if args.use_alpha:
          cmd += ["alpha"]
     cmd += [
         "compute",
         "tpus",
         "tpu-vm",
         "ssh",
         args.tpu_name,
         "--zone",
         args.tpu_zone,
         "--command",
         args.command,
         "--worker",
         "all",
     ]
     if args.debug:
          print(F"""Running {" ".join(__A       )}"""       )
          return
     subprocess.run(__A       )
     print('''Successfully setup pod.'''       )


def    A						()   ->      Optional[int]:







     """simple docstring"""


     UpperCAmelCase_          =							tpu_command_parser()
     UpperCAmelCase_          =							parser.parse_args()

     tpu_command_launcher(__A       )












import random
import timeit
from functools import wraps
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_tf_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_tf_available():
    import tensorflow as tf
    from tensorflow.python.framework.errors_impl import ResourceExhaustedError

    from .benchmark_args_tf import TensorFlowBenchmarkArguments

if is_pyanvml_available():
    import pyanvml.pyanvml as nvml

snake_case_ : Tuple    		=       logging.get_logger(__name__)


def    A						(__A   :    bool   ,				__A   :    bool       )   ->      Optional[Any]:







     """simple docstring"""


     def run_func(__A   :    Optional[Any]       ):
          @wraps(__A       )
          def run_in_eager_mode(*__A   :    Dict   ,				**__A   :    List[Any]       ):
               return func(*__A   ,				**__A       )

          @wraps(__A       )
          @tf.function(experimental_compile=__A       )
          def run_in_graph_mode(*__A   :    Optional[Any]   ,				**__A   :    Any       ):
               return func(*__A   ,				**__A       )

          if do_eager_mode is True:
               if use_xla is not False:
                    raise ValueError(
                        '''Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`.'''       )
               return run_in_eager_mode
          else:
               return run_in_graph_mode

     return run_func


def    A						(__A   :    int   ,				__A   :    int   ,				__A   :    int       )   ->      ["tf.Tensor"]:







     """simple docstring"""


     UpperCAmelCase_          =							random.Random()
     UpperCAmelCase_          =							[rng.randint(0   ,				vocab_size - 1       ) for i in range(batch_size * sequence_length       )]
     return tf.constant(__A   ,				shape=(batch_size, sequence_length)   ,				dtype=tf.intaa       )







class __snake_case			(			a		):
   UpperCAmelCase__  :    TensorFlowBenchmarkArguments
   UpperCAmelCase__  :    PretrainedConfig
   UpperCAmelCase__  :    str          =  "TensorFlow"







   @property
   def lowerCamelCase						(      self		:						List[str]):

        """simple docstring"""


        return tf.__version__







   def lowerCamelCase						(      self		:						Dict					,						_snake_case		:						str					,						_snake_case		:						int					,						_snake_case		:						int):

        """simple docstring"""


        UpperCAmelCase_          =							self.args.strategy
        if strategy is None:
             raise ValueError('''A device strategy has to be initialized before using TensorFlow.''')
        UpperCAmelCase_          =							self._prepare_inference_func(_snake_case					,						_snake_case					,						_snake_case)
        return self._measure_speed(_inference)







   def lowerCamelCase						(      self		:						Any					,						_snake_case		:						str					,						_snake_case		:						int					,						_snake_case		:						int):

        """simple docstring"""


        UpperCAmelCase_          =							self.args.strategy
        if strategy is None:
             raise ValueError('''A device strategy has to be initialized before using TensorFlow.''')
        UpperCAmelCase_          =							self._prepare_train_func(_snake_case					,						_snake_case					,						_snake_case)
        return self._measure_speed(_train)







   def lowerCamelCase						(      self		:						Any					,						_snake_case		:						str					,						_snake_case		:						int					,						_snake_case		:						int):

        """simple docstring"""


        if self.args.is_gpu:
             tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]					,						_snake_case)
        UpperCAmelCase_          =							self.args.strategy
        if strategy is None:
             raise ValueError('''A device strategy has to be initialized before using TensorFlow.''')
        UpperCAmelCase_          =							self._prepare_inference_func(_snake_case					,						_snake_case					,						_snake_case)
        return self._measure_memory(_inference)







   def lowerCamelCase						(      self		:						Optional[Any]					,						_snake_case		:						str					,						_snake_case		:						int					,						_snake_case		:						int):

        """simple docstring"""


        if self.args.is_gpu:
             tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]					,						_snake_case)
        UpperCAmelCase_          =							self.args.strategy
        if strategy is None:
             raise ValueError('''A device strategy has to be initialized before using TensorFlow.''')

        UpperCAmelCase_          =							self._prepare_train_func(_snake_case					,						_snake_case					,						_snake_case)
        return self._measure_memory(_train)







   def lowerCamelCase						(      self		:						Optional[int]					,						_snake_case		:						str					,						_snake_case		:						int					,						_snake_case		:						int):

        """simple docstring"""


        UpperCAmelCase_          =							self.config_dict[model_name]

        if self.args.fpaa:
             raise NotImplementedError('''Mixed precision is currently not supported.''')

        UpperCAmelCase_          =							(
            hasattr(_snake_case					,						'''architectures''')
            and isinstance(config.architectures					,						_snake_case)
            and len(config.architectures) > 0
        )
        if not self.args.only_pretrain_model and has_model_class_in_config:
             try:
                  UpperCAmelCase_          =							'''TF''' + config.architectures[0]  # prepend 'TF' for tensorflow model
                  UpperCAmelCase_          =							__import__('''transformers'''					,						fromlist=[model_class])
                  UpperCAmelCase_          =							getattr(_snake_case					,						_snake_case)
                  UpperCAmelCase_          =							model_cls(_snake_case)
             except ImportError:
                  raise ImportError(
                      F"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
                      ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.''')
        else:
             UpperCAmelCase_          =							TF_MODEL_MAPPING[config.__class__](_snake_case)

        # encoder-decoder has vocab size saved differently
        UpperCAmelCase_          =							config.vocab_size if hasattr(_snake_case					,						'''vocab_size''') else config.encoder.vocab_size
        UpperCAmelCase_          =							random_input_ids(_snake_case					,						_snake_case					,						_snake_case)

        @run_with_tf_optimizations(self.args.eager_mode					,						self.args.use_xla)
        def encoder_decoder_forward():
             return model(_snake_case					,						decoder_input_ids=_snake_case					,						training=_snake_case)

        @run_with_tf_optimizations(self.args.eager_mode					,						self.args.use_xla)
        def encoder_forward():
             return model(_snake_case					,						training=_snake_case)

        UpperCAmelCase_          =							encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

        return _inference







   def lowerCamelCase						(      self		:						Optional[Any]					,						_snake_case		:						str					,						_snake_case		:						int					,						_snake_case		:						int):

        """simple docstring"""


        UpperCAmelCase_          =							self.config_dict[model_name]

        if self.args.eager_mode is not False:
             raise ValueError('''Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`.''')

        if self.args.fpaa:
             raise NotImplementedError('''Mixed precision is currently not supported.''')

        UpperCAmelCase_          =							(
            hasattr(_snake_case					,						'''architectures''')
            and isinstance(config.architectures					,						_snake_case)
            and len(config.architectures) > 0
        )
        if not self.args.only_pretrain_model and has_model_class_in_config:
             try:
                  UpperCAmelCase_          =							'''TF''' + config.architectures[0]  # prepend 'TF' for tensorflow model
                  UpperCAmelCase_          =							__import__('''transformers'''					,						fromlist=[model_class])
                  UpperCAmelCase_          =							getattr(_snake_case					,						_snake_case)
                  UpperCAmelCase_          =							model_cls(_snake_case)
             except ImportError:
                  raise ImportError(
                      F"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
                      ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.''')
        else:
             UpperCAmelCase_          =							TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](_snake_case)

        # encoder-decoder has vocab size saved differently
        UpperCAmelCase_          =							config.vocab_size if hasattr(_snake_case					,						'''vocab_size''') else config.encoder.vocab_size
        UpperCAmelCase_          =							random_input_ids(_snake_case					,						_snake_case					,						_snake_case)

        @run_with_tf_optimizations(self.args.eager_mode					,						self.args.use_xla)
        def encoder_decoder_train():
             UpperCAmelCase_          =							model(_snake_case					,						decoder_input_ids=_snake_case					,						labels=_snake_case					,						training=_snake_case)[0]
             UpperCAmelCase_          =							tf.gradients(_snake_case					,						model.trainable_variables)
             return gradients

        @run_with_tf_optimizations(self.args.eager_mode					,						self.args.use_xla)
        def encoder_train():
             UpperCAmelCase_          =							model(_snake_case					,						labels=_snake_case					,						training=_snake_case)[0]
             UpperCAmelCase_          =							tf.gradients(_snake_case					,						model.trainable_variables)
             return gradients

        UpperCAmelCase_          =							encoder_decoder_train if config.is_encoder_decoder else encoder_train

        return _train







   def lowerCamelCase						(      self		:						Any					,						_snake_case		:						Optional[Any]):

        """simple docstring"""


        with self.args.strategy.scope():
             try:
                  if self.args.is_tpu or self.args.use_xla:
                       # run additional 10 times to stabilize compilation for tpu
                       logger.info('''Do inference on TPU. Running model 5 times to stabilize compilation''')
                       timeit.repeat(_snake_case					,						repeat=1					,						number=5)

                  # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
                  UpperCAmelCase_          =							timeit.repeat(
                      _snake_case					,						repeat=self.args.repeat					,						number=10					,						)

                  return min(_snake_case) / 1_0.0
             except ResourceExhaustedError as e:
                  self.print_fn(F"""Doesn't fit on GPU. {e}""")







   def lowerCamelCase						(      self		:						Dict					,						_snake_case		:						Callable[[], None]):

        """simple docstring"""


        logger.info(
            '''Note that TensorFlow allocates more memory than '''
            '''it might need to speed up computation. '''
            '''The memory reported here corresponds to the memory '''
            '''reported by `nvidia-smi`, which can vary depending '''
            '''on total available memory on the GPU that is used.''')
        with self.args.strategy.scope():
             try:
                  if self.args.trace_memory_line_by_line:
                       if not self.args.eager_mode:
                            raise ValueError(
                                '''`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory'''
                                ''' consumption line by line.''')
                       UpperCAmelCase_          =							start_memory_tracing('''transformers''')

                  if self.args.is_tpu:
                       # tpu
                       raise NotImplementedError(
                           '''Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking'''
                           ''' with `args.memory=False`''')
                  elif self.args.is_gpu:
                       # gpu
                       if not is_pyanvml_available():
                            logger.warning(
                                '''py3nvml not installed, we won\'t log GPU memory usage. '''
                                '''Install py3nvml (pip install py3nvml) to log information about GPU.''')
                            UpperCAmelCase_          =							'''N/A'''
                       else:
                            logger.info(
                                '''Measuring total GPU usage on GPU device. Make sure to not have additional processes'''
                                ''' running on the same GPU.''')
                            # init nvml
                            nvml.nvmlInit()
                            func()
                            UpperCAmelCase_          =							nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx)
                            UpperCAmelCase_          =							nvml.nvmlDeviceGetMemoryInfo(_snake_case)
                            UpperCAmelCase_          =							meminfo.used
                            UpperCAmelCase_          =							Memory(_snake_case)
                            # shutdown nvml
                            nvml.nvmlShutdown()
                  else:
                       # cpu
                       if self.args.trace_memory_line_by_line:
                            logger.info(
                                '''When enabling line by line tracing, the max peak memory for CPU is inaccurate in'''
                                ''' TensorFlow.''')
                            UpperCAmelCase_          =							None
                       else:
                            UpperCAmelCase_          =							measure_peak_memory_cpu(_snake_case)
                            UpperCAmelCase_          =							Memory(_snake_case) if isinstance(_snake_case					,						_snake_case) else memory_bytes
                  if self.args.trace_memory_line_by_line:
                       UpperCAmelCase_          =							stop_memory_tracing(_snake_case)
                       if memory is None:
                            UpperCAmelCase_          =							summary.total
                  else:
                       UpperCAmelCase_          =							None

                  return memory, summary
             except ResourceExhaustedError as e:
                  self.print_fn(F"""Doesn't fit on GPU. {e}""")
                  return "N/A", None






