


"""simple docstring"""






from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


_lowerCamelCase  =	logging.get_logger(__name__)

_lowerCamelCase  =	{
    "gpt2": "https://huggingface.co/gpt2/resolve/main/config.json",
    "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/config.json",
    "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/config.json",
    "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/config.json",
    "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/config.json",
}


class       lowerCamelCase_		(      UpperCAmelCase_ ):





				"""simple docstring"""







				_lowerCAmelCase   :   Tuple   				=       'gpt2'
				_lowerCAmelCase   :   Tuple   				=       ['past_key_values']
				_lowerCAmelCase   :   List[Any]   				=       {
				    'hidden_size': 'n_embd',
				    'max_position_embeddings': 'n_positions',
				    'num_attention_heads': 'n_head',
				    'num_hidden_layers': 'n_layer',
				}



				def __init__(    self							,		UpperCAmelCase__=5_0257							,		UpperCAmelCase__=1024							,		UpperCAmelCase__=768							,		UpperCAmelCase__=12							,		UpperCAmelCase__=12							,		UpperCAmelCase__=None							,		UpperCAmelCase__="gelu_new"							,		UpperCAmelCase__=0.1							,		UpperCAmelCase__=0.1							,		UpperCAmelCase__=0.1							,		UpperCAmelCase__=1e-5							,		UpperCAmelCase__=0.02							,		UpperCAmelCase__="cls_index"							,		UpperCAmelCase__=True							,		UpperCAmelCase__=None							,		UpperCAmelCase__=True							,		UpperCAmelCase__=0.1							,		UpperCAmelCase__=True							,		UpperCAmelCase__=True							,		UpperCAmelCase__=5_0256							,		UpperCAmelCase__=5_0256							,		UpperCAmelCase__=False							,		UpperCAmelCase__=False							,		**UpperCAmelCase__							,		):
								SCREAMING_SNAKE_CASE__  				=					vocab_size
								SCREAMING_SNAKE_CASE__  				=					n_positions
								SCREAMING_SNAKE_CASE__  				=					n_embd
								SCREAMING_SNAKE_CASE__  				=					n_layer
								SCREAMING_SNAKE_CASE__  				=					n_head
								SCREAMING_SNAKE_CASE__  				=					n_inner
								SCREAMING_SNAKE_CASE__  				=					activation_function
								SCREAMING_SNAKE_CASE__  				=					resid_pdrop
								SCREAMING_SNAKE_CASE__  				=					embd_pdrop
								SCREAMING_SNAKE_CASE__  				=					attn_pdrop
								SCREAMING_SNAKE_CASE__  				=					layer_norm_epsilon
								SCREAMING_SNAKE_CASE__  				=					initializer_range
								SCREAMING_SNAKE_CASE__  				=					summary_type
								SCREAMING_SNAKE_CASE__  				=					summary_use_proj
								SCREAMING_SNAKE_CASE__  				=					summary_activation
								SCREAMING_SNAKE_CASE__  				=					summary_first_dropout
								SCREAMING_SNAKE_CASE__  				=					summary_proj_to_labels
								SCREAMING_SNAKE_CASE__  				=					scale_attn_weights
								SCREAMING_SNAKE_CASE__  				=					use_cache
								SCREAMING_SNAKE_CASE__  				=					scale_attn_by_inverse_layer_idx
								SCREAMING_SNAKE_CASE__  				=					reorder_and_upcast_attn

								SCREAMING_SNAKE_CASE__  				=					bos_token_id
								SCREAMING_SNAKE_CASE__  				=					eos_token_id

								super().__init__(bos_token_id=_lowercase							,		eos_token_id=_lowercase							,		**_lowercase  )


class       lowerCamelCase_		(      UpperCAmelCase_ ):





				"""simple docstring"""







				def __init__(    self							,		UpperCAmelCase__							,		UpperCAmelCase__ = "default"							,		UpperCAmelCase__ = None							,		UpperCAmelCase__ = False							,		):
								super().__init__(_lowercase							,		task=_lowercase							,		patching_specs=_lowercase							,		use_past=_lowercase  )
								if not getattr(self._config							,		"pad_token_id"							,		_lowercase  ):
												# TODO: how to do that better?
												SCREAMING_SNAKE_CASE__  				=					0



				@property
				def 	lowerCAmelCase__		(    self  ):
								SCREAMING_SNAKE_CASE__  				=					OrderedDict({"input_ids": {0: "batch", 1: "sequence"}}  )
								if self.use_past:
												self.fill_with_past_key_values_(_lowercase							,		direction="inputs"  )
												SCREAMING_SNAKE_CASE__  				=					{0: 'batch', 1: 'past_sequence + sequence'}
								else:
												SCREAMING_SNAKE_CASE__  				=					{0: 'batch', 1: 'sequence'}

								return common_inputs



				@property
				def 	lowerCAmelCase__		(    self  ):
								return self._config.n_layer



				@property
				def 	lowerCAmelCase__		(    self  ):
								return self._config.n_head



				def 	lowerCAmelCase__		(    self							,		UpperCAmelCase__							,		UpperCAmelCase__ = -1							,		UpperCAmelCase__ = -1							,		UpperCAmelCase__ = False							,		UpperCAmelCase__ = None							,		):
								SCREAMING_SNAKE_CASE__  				=					super(_lowercase							,		self  ).generate_dummy_inputs(
								    _lowercase							,		batch_size=_lowercase							,		seq_length=_lowercase							,		is_pair=_lowercase							,		framework=_lowercase  )

								# We need to order the input in the way they appears in the forward()
								SCREAMING_SNAKE_CASE__  				=					OrderedDict({"input_ids": common_inputs["input_ids"]}  )

								# Need to add the past_keys
								if self.use_past:
												if not is_torch_available():
																raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."  )
												else:
																import torch

																SCREAMING_SNAKE_CASE__  				=					common_inputs['input_ids'].shape
																# Not using the same length for past_key_values
																SCREAMING_SNAKE_CASE__  				=					seqlen + 2
																SCREAMING_SNAKE_CASE__  				=					(
																    batch,
																    self.num_attention_heads,
																    past_key_values_length,
																    self._config.hidden_size // self.num_attention_heads,
																)
																SCREAMING_SNAKE_CASE__  				=					[
																    (torch.zeros(_lowercase  ), torch.zeros(_lowercase  )) for _ in range(self.num_layers  )
																]

								SCREAMING_SNAKE_CASE__  				=					common_inputs['attention_mask']
								if self.use_past:
												SCREAMING_SNAKE_CASE__  				=					ordered_inputs['attention_mask'].dtype
												SCREAMING_SNAKE_CASE__  				=					torch.cat(
												    [ordered_inputs["attention_mask"], torch.ones(_lowercase							,		_lowercase							,		dtype=_lowercase  )]							,		dim=1  )

								return ordered_inputs






				@property
				def 	lowerCAmelCase__		(    self  ):
								return 13








'''simple docstring'''

import importlib
import sys
from argparse import REMAINDER, ArgumentParser
from pathlib import Path

import torch_xla.distributed.xla_multiprocessing as xmp






def 						_UpperCAmelCase     (	)      ->      Optional[Any]:





						"""simple docstring"""
						lowercase_   : List[str]   	=      ArgumentParser(
						    description=(
						        'PyTorch TPU distributed training launch helper utility that will spawn up multiple distributed processes'
						    )				)

						# Optional arguments for the launch helper
						parser.add_argument('--num_cores'    ,      type=a    ,      default=1    ,      help='Number of TPU cores to use (1 or 8).'				)

						# positional
						parser.add_argument(
						    'training_script'    ,      type=a    ,      help=(
						        'The full path to the single TPU training '
						        'program/script to be launched in parallel, '
						        'followed by all the arguments for the '
						        'training script'
						    )    ,      )

						# rest from the training program
						parser.add_argument('training_script_args'    ,      nargs=a				)

						return parser.parse_args()






def 						_UpperCAmelCase     (	)      ->      str:





						"""simple docstring"""
						lowercase_   : Optional[int]   	=      parse_args()

						# Import training_script as a module.
						lowercase_   : str   	=      Path(args.training_script				)
						sys.path.append(str(script_fpath.parent.resolve()				)				)
						lowercase_   : int   	=      script_fpath.stem
						lowercase_   : List[Any]   	=      importlib.import_module(a				)

						# Patch sys.argv
						lowercase_   : List[str]   	=      [args.training_script] + args.training_script_args + ['--tpu_num_cores', str(args.num_cores				)]

						xmp.spawn(mod._mp_fn    ,      args=()    ,      nprocs=args.num_cores				)


if __name__ == "__main__":
						main()
