


# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import subprocess

from packaging.version import Version, parse

from accelerate.commands.config.config_args import default_config_file, load_config_from_file


UpperCAmelCase_      : Dict  					=       'Run commands across TPU VMs for initial setup before running `accelerate launch`.'







def        SCREAMING_SNAKE_CASE_						(			__A :			int=None				)	->		Optional[Any]:

       """simple docstring"""

       if subparsers is not None:
              a_					:		Optional[Any]							=  subparsers.add_parser('tpu-config'  ,    description=_description				)
       else:
              a_					:		Dict							=  argparse.ArgumentParser('Accelerate tpu-config command'  ,    description=_description				)
       # Core arguments
       a_					:		Optional[int]							=  parser.add_argument_group(
           'Config Arguments'  ,    'Arguments that can be configured through `accelerate config`.'				)
       config_args.add_argument(
           '--config_file'  ,    type=__A  ,    default=__A  ,    help='Path to the config file to use for accelerate.'  ,    )
       config_args.add_argument(
           '--tpu_name'  ,    default=__A  ,    help='The name of the TPU to use. If not specified, will use the TPU specified in the config file.'  ,    )
       config_args.add_argument(
           '--tpu_zone'  ,    default=__A  ,    help='The zone of the TPU to use. If not specified, will use the zone specified in the config file.'  ,    )
       a_					:		str							=  parser.add_argument_group('TPU Arguments'  ,    'Arguments for options ran inside the TPU.'				)
       pod_args.add_argument(
           '--use_alpha'  ,    action='store_true'  ,    help='Whether to use `gcloud alpha` when running the TPU training script instead of `gcloud`.'  ,    )
       pod_args.add_argument(
           '--command_file'  ,    default=__A  ,    help='The path to the file containing the commands to run on the pod on startup.'  ,    )
       pod_args.add_argument(
           '--command'  ,    action='append'  ,    nargs='+'  ,    help='A command to run on the pod. Can be passed multiple times.'  ,    )
       pod_args.add_argument(
           '--install_accelerate'  ,    action='store_true'  ,    help='Whether to install accelerate on the pod. Defaults to False.'  ,    )
       pod_args.add_argument(
           '--accelerate_version'  ,    default='latest'  ,    help='The version of accelerate to install on the pod. If not specified, will use the latest pypi version. Specify \'dev\' to install from GitHub.'  ,    )
       pod_args.add_argument(
           '--debug'  ,    action='store_true'  ,    help='If set, will print the command that would be run instead of running it.'				)

       if subparsers is not None:
              parser.set_defaults(func=__A				)
       return parser







def        SCREAMING_SNAKE_CASE_						(			__A :			int				)	->		str:

       """simple docstring"""

       a_					:		Optional[Any]							=  None

       # Get the default from the config file if it exists.
       if args.config_file is not None or os.path.isfile(__A				):
              a_					:		Optional[int]							=  load_config_from_file(args.config_file				)
              if not args.command_file and defaults.command_file is not None and not args.command:
                     a_					:		Dict							=  defaults.command_file
              if not args.command and defaults.commands is not None:
                     a_					:		List[Any]							=  defaults.commands
              if not args.tpu_name:
                     a_					:		Union[str, Any]							=  defaults.tpu_name
              if not args.tpu_zone:
                     a_					:		Union[str, Any]							=  defaults.tpu_zone
       if args.accelerate_version == "dev":
              a_					:		int							=  'git+https://github.com/huggingface/accelerate.git'
       elif args.accelerate_version == "latest":
              a_					:		Any							=  'accelerate -U'
       elif isinstance(parse(args.accelerate_version				)  ,    __A				):
              a_					:		Optional[Any]							=  F"""accelerate=={args.accelerate_version}"""

       if not args.command_file and not args.command:
              raise ValueError('You must specify either a command file or a command to run on the pod.'				)

       if args.command_file:
              with open(args.command_file  ,    'r'				) as f:
                     a_					:		Union[str, Any]							=  [f.read().splitlines()]

                      # To turn list of lists into list of strings
       if isinstance(args.command[0]  ,    __A				):
              a_					:		List[str]							=  [line for cmd in args.command for line in cmd]
       # Default to the shared folder and install accelerate
       a_					:		Union[str, Any]							=  ['cd /usr/share']
       if args.install_accelerate:
              new_cmd += [F"""pip install {args.accelerate_version}"""]
       new_cmd += args.command
       a_					:		List[str]							=  '; '.join(__A				)

       # Then send it to gcloud
       # Eventually try to use google-api-core to do this instead of subprocess
       a_					:		List[str]							=  ['gcloud']
       if args.use_alpha:
              cmd += ["alpha"]
       cmd += [
           "compute",
           "tpus",
           "tpu-vm",
           "ssh",
           args.tpu_name,
           "--zone",
           args.tpu_zone,
           "--command",
           args.command,
           "--worker",
           "all",
       ]
       if args.debug:
              print(F"""Running {" ".join(__A				)}"""				)
              return
       subprocess.run(__A				)
       print('Successfully setup pod.'				)







def        SCREAMING_SNAKE_CASE_						(			)	->		Any:

       """simple docstring"""

       a_					:		Optional[int]							=  tpu_command_parser()
       a_					:		Dict							=  parser.parse_args()

       tpu_command_launcher(__A				)










from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxSeqaSeqConfigWithPast
from ...utils import logging


UpperCAmelCase__				   =		logging.get_logger(__name__)

UpperCAmelCase__				   =		{
    "t5-small": "https://huggingface.co/t5-small/resolve/main/config.json",
    "t5-base": "https://huggingface.co/t5-base/resolve/main/config.json",
    "t5-large": "https://huggingface.co/t5-large/resolve/main/config.json",
    "t5-3b": "https://huggingface.co/t5-3b/resolve/main/config.json",
    "t5-11b": "https://huggingface.co/t5-11b/resolve/main/config.json",
}






class 		lowercase_					(	lowercase   ):






      '''simple docstring'''




      __snake_case       =  '''t5'''
      __snake_case       =  ['''past_key_values''']
      __snake_case       =  {'''hidden_size''': '''d_model''', '''num_attention_heads''': '''num_heads''', '''num_hidden_layers''': '''num_layers'''}







      def __init__(      self      :						Optional[Any] ,					__UpperCAmelCase      :						Optional[Any]=32_128 ,					__UpperCAmelCase      :						List[Any]=512 ,					__UpperCAmelCase      :						Dict=64 ,					__UpperCAmelCase      :						Tuple=2_048 ,					__UpperCAmelCase      :						int=6 ,					__UpperCAmelCase      :						Optional[int]=None ,					__UpperCAmelCase      :						Optional[int]=8 ,					__UpperCAmelCase      :						str=32 ,					__UpperCAmelCase      :						Tuple=128 ,					__UpperCAmelCase      :						Optional[Any]=0.1 ,					__UpperCAmelCase      :						int=1e-6 ,					__UpperCAmelCase      :						int=1.0 ,					__UpperCAmelCase      :						List[str]="relu" ,					__UpperCAmelCase      :						int=True ,					__UpperCAmelCase      :						int=True ,					__UpperCAmelCase      :						List[Any]=0 ,					__UpperCAmelCase      :						int=1 ,					**__UpperCAmelCase      :						str ,					)    ->Optional[Any]:




       """simple docstring"""


       a		  =      vocab_size
       a		  =      d_model
       a		  =      d_kv
       a		  =      d_ff
       a		  =      num_layers
       a		  =      (
           num_decoder_layers if num_decoder_layers is not None else self.num_layers
       )  # default = symmetry
       a		  =      num_heads
       a		  =      relative_attention_num_buckets
       a		  =      relative_attention_max_distance
       a		  =      dropout_rate
       a		  =      layer_norm_epsilon
       a		  =      initializer_factor
       a		  =      feed_forward_proj
       a		  =      use_cache

       a		  =      self.feed_forward_proj.split('''-'''  )
       a		  =      act_info[-1]
       a		  =      act_info[0] == '''gated'''

       if len(__UpperCAmelCase  ) > 1 and act_info[0] != "gated" or len(__UpperCAmelCase  ) > 2:
        raise ValueError(
            F"""`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."""
            '''Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. '''
            '''\'gated-gelu\' or \'relu\''''  )

       # for backwards compatibility
       if feed_forward_proj == "gated-gelu":
        a		  =      '''gelu_new'''

       super().__init__(
           pad_token_id=__UpperCAmelCase ,					eos_token_id=__UpperCAmelCase ,					is_encoder_decoder=__UpperCAmelCase ,					**__UpperCAmelCase ,					)






class 		lowercase_					(	lowercase   ):






      '''simple docstring'''




      @property
      def       __lowerCAmelCase					(      self      :						Optional[Any]  )    ->Mapping[str, Mapping[int, str]]:




       """simple docstring"""


       a		  =      {
           '''input_ids''': {0: '''batch''', 1: '''encoder_sequence'''},
           '''attention_mask''': {0: '''batch''', 1: '''encoder_sequence'''},
       }
       if self.use_past:
        a		  =      '''past_encoder_sequence + sequence'''
        a		  =      {0: '''batch'''}
        a		  =      {0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
       else:
        a		  =      {0: '''batch''', 1: '''decoder_sequence'''}
        a		  =      {0: '''batch''', 1: '''decoder_sequence'''}

       if self.use_past:
        self.fill_with_past_key_values_(__UpperCAmelCase ,					direction='''inputs'''  )

       return common_inputs







      @property
      def       __lowerCAmelCase					(      self      :						Union[str, Any]  )    ->int:




       """simple docstring"""


       return 13


