


"""simple docstring"""




from collections import OrderedDict
from typing import TYPE_CHECKING, Any, List, Mapping, Optional

from packaging import version


if TYPE_CHECKING:
	from ... import PreTrainedTokenizer, TensorType

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import is_torch_available, logging


__UpperCamelCase						=      logging.get_logger(__name__)

__UpperCamelCase						=      {
    '''bigscience/bloom''': '''https://huggingface.co/bigscience/bloom/resolve/main/config.json''',
    '''bigscience/bloom-560m''': '''https://huggingface.co/bigscience/bloom-560m/blob/main/config.json''',
    '''bigscience/bloom-1b1''': '''https://huggingface.co/bigscience/bloom-1b1/blob/main/config.json''',
    '''bigscience/bloom-1b7''': '''https://huggingface.co/bigscience/bloom-1b7/blob/main/config.json''',
    '''bigscience/bloom-3b''': '''https://huggingface.co/bigscience/bloom-3b/blob/main/config.json''',
    '''bigscience/bloom-7b1''': '''https://huggingface.co/bigscience/bloom-7b1/blob/main/config.json''',
}
class 					lowerCAmelCase	(					snake_case_    ):




		'''simple docstring'''







		SCREAMING_SNAKE_CASE_						:     Any								=     """bloom"""
		SCREAMING_SNAKE_CASE_						:     Tuple								=     ["""past_key_values"""]
		SCREAMING_SNAKE_CASE_						:     str								=     {
		    """num_hidden_layers""": """n_layer""",
		    """num_attention_heads""": """n_head""",
		}







		def __init__(	self						,       lowerCAmelCase__=250_880						,       lowerCAmelCase__=64						,       lowerCAmelCase__=2						,       lowerCAmelCase__=8						,       lowerCAmelCase__=1e-5						,       lowerCAmelCase__=0.02						,       lowerCAmelCase__=True						,       lowerCAmelCase__=1						,       lowerCAmelCase__=2						,       lowerCAmelCase__=False						,       lowerCAmelCase__=0.0						,       lowerCAmelCase__=0.0						,       lowerCAmelCase__=1						,       lowerCAmelCase__=False						,       **lowerCAmelCase__						,       )     ->							Tuple:
								SCREAMING_SNAKE_CASE													=					vocab_size
								# Backward compatibility with n_embed kwarg
								SCREAMING_SNAKE_CASE													=					kwargs.pop('n_embed'						,       lowerCAmelCase__				)
								SCREAMING_SNAKE_CASE													=					hidden_size if n_embed is None else n_embed
								SCREAMING_SNAKE_CASE													=					n_layer
								SCREAMING_SNAKE_CASE													=					n_head
								SCREAMING_SNAKE_CASE													=					layer_norm_epsilon
								SCREAMING_SNAKE_CASE													=					initializer_range
								SCREAMING_SNAKE_CASE													=					use_cache
								SCREAMING_SNAKE_CASE													=					pretraining_tp
								SCREAMING_SNAKE_CASE													=					apply_residual_connection_post_layernorm
								SCREAMING_SNAKE_CASE													=					hidden_dropout
								SCREAMING_SNAKE_CASE													=					attention_dropout

								SCREAMING_SNAKE_CASE													=					bos_token_id
								SCREAMING_SNAKE_CASE													=					eos_token_id
								SCREAMING_SNAKE_CASE													=					slow_but_exact

								super().__init__(bos_token_id=lowerCAmelCase__						,       eos_token_id=lowerCAmelCase__						,       **lowerCAmelCase__				)





class 					lowerCAmelCase	(					snake_case_    ):




		'''simple docstring'''







		SCREAMING_SNAKE_CASE_						:     Tuple								=     version.parse("""1.12"""    )







		def __init__(	self						,       lowerCAmelCase__						,       lowerCAmelCase__ = "default"						,       lowerCAmelCase__ = None						,       lowerCAmelCase__ = False						,       )     ->							str:
								super().__init__(lowerCAmelCase__						,       task=lowerCAmelCase__						,       patching_specs=lowerCAmelCase__						,       use_past=lowerCAmelCase__				)
								if not getattr(self._config						,       'pad_token_id'						,       lowerCAmelCase__				):
														# TODO: how to do that better?
														SCREAMING_SNAKE_CASE													=					0







		@property
		def 							__A						(	self				)     ->							Mapping[str, Mapping[int, str]]:
								SCREAMING_SNAKE_CASE													=					OrderedDict({'input_ids': {0: 'batch', 1: 'sequence'}}				)
								if self.use_past:
														# BLOOM stores values on dynamic axis 2. For more details see: https://github.com/huggingface/transformers/pull/18344
														self.fill_with_past_key_values_(lowerCAmelCase__						,       direction='inputs'						,       inverted_values_shape=lowerCAmelCase__				)
														SCREAMING_SNAKE_CASE													=					{0: 'batch', 1: 'past_sequence + sequence'}
								else:
														SCREAMING_SNAKE_CASE													=					{0: 'batch', 1: 'sequence'}

								return common_inputs







		@property
		def 							__A						(	self				)     ->							int:
								return self._config.n_layer







		@property
		def 							__A						(	self				)     ->							int:
								return self._config.n_head







		@property
		def 							__A						(	self				)     ->							float:
								return 1e-3







		def 							__A						(	self						,       lowerCAmelCase__						,       lowerCAmelCase__ = -1						,       lowerCAmelCase__ = -1						,       lowerCAmelCase__ = False						,       lowerCAmelCase__ = None						,       )     ->							Mapping[str, Any]:
								SCREAMING_SNAKE_CASE													=					super(lowerCAmelCase__						,       self				).generate_dummy_inputs(
								    lowerCAmelCase__						,       batch_size=lowerCAmelCase__						,       seq_length=lowerCAmelCase__						,       is_pair=lowerCAmelCase__						,       framework=lowerCAmelCase__				)

								# We need to order the input in the way they appears in the forward()
								SCREAMING_SNAKE_CASE													=					OrderedDict({'input_ids': common_inputs['input_ids']}				)

								# Need to add the past_keys
								if self.use_past:
														if not is_torch_available():
																				raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'				)
														else:
																				import torch

																				SCREAMING_SNAKE_CASE							,				SCREAMING_SNAKE_CASE													=					common_inputs['input_ids'].shape
																				# Not using the same length for past_key_values
																				SCREAMING_SNAKE_CASE													=					seqlen + 2
																				SCREAMING_SNAKE_CASE													=					self._config.hidden_size // self.num_attention_heads
																				SCREAMING_SNAKE_CASE													=					(
																				    batch * self.num_attention_heads,
																				    head_dim,
																				    past_key_values_length,
																				)
																				SCREAMING_SNAKE_CASE													=					(
																				    batch * self.num_attention_heads,
																				    past_key_values_length,
																				    head_dim,
																				)
																				SCREAMING_SNAKE_CASE													=					[
																				    (torch.zeros(lowerCAmelCase__				), torch.zeros(lowerCAmelCase__				)) for _ in range(self.num_layers				)
																				]

								SCREAMING_SNAKE_CASE													=					common_inputs['attention_mask']
								if self.use_past:
														SCREAMING_SNAKE_CASE													=					ordered_inputs['attention_mask'].dtype
														SCREAMING_SNAKE_CASE													=					torch.cat(
														    [ordered_inputs['attention_mask'], torch.ones(lowerCAmelCase__						,       lowerCAmelCase__						,       dtype=lowerCAmelCase__				)]						,       dim=1				)

								return ordered_inputs







		@property
		def 							__A						(	self				)     ->							int:
								return 13









'''simple docstring'''

import argparse
import shlex

import runhouse as rh


if __name__ == "__main__":
      # Refer to https://runhouse-docs.readthedocs-hosted.com/en/latest/api/python/cluster.html#hardware-setup for cloud access
      # setup instructions, if using on-demand hardware

      # If user passes --user <user> --host <host> --key_path <key_path> <example> <args>, fill them in as BYO cluster
      # If user passes --instance <instance> --provider <provider> <example> <args>, fill them in as on-demand cluster
      # Throw an error if user passes both BYO and on-demand cluster args
      # Otherwise, use default values
      lowercase_					= argparse.ArgumentParser()
      parser.add_argument("""--user""", type=str, default="""ubuntu""")
      parser.add_argument("""--host""", type=str, default="""localhost""")
      parser.add_argument("""--key_path""", type=str, default=None)
      parser.add_argument("""--instance""", type=str, default="""V100:1""")
      parser.add_argument("""--provider""", type=str, default="""cheapest""")
      parser.add_argument("""--use_spot""", type=bool, default=False)
      parser.add_argument("""--example""", type=str, default="""pytorch/text-generation/run_generation.py""")
      lowercase_				,			lowercase_					= parser.parse_known_args()
      if args.host != "localhost":
            if args.instance != "V100:1" or args.provider != "cheapest":
                  raise ValueError("""Cannot specify both BYO and on-demand cluster args""")
            lowercase_					= rh.cluster(
                name="""rh-cluster""", ips=[args.host], ssh_creds={"""ssh_user""": args.user, """ssh_private_key""": args.key_path}
            )
      else:
            lowercase_					= rh.cluster(
                name="""rh-cluster""", instance_type=args.instance, provider=args.provider, use_spot=args.use_spot
            )
      lowercase_					= args.example.rsplit("""/""", 1)[0]

      # Set up remote environment
      cluster.install_packages(["""pip:./"""])  # Installs transformers from local source
      # Note transformers is copied into the home directory on the remote machine, so we can install from there
      cluster.run([f"""pip install -r transformers/examples/{example_dir}/requirements.txt"""])
      cluster.run(["""pip install torch --upgrade --extra-index-url https://download.pytorch.org/whl/cu117"""])

      # Run example. You can bypass the CLI wrapper and paste your own code here.
      cluster.run([f"""python transformers/examples/{args.example} {" ".join(shlex.quote(arg) for arg in unknown)}"""])

      # Alternatively, we can just import and run a training function (especially if there's no wrapper CLI):
      # from my_script... import train
      # reqs = ['pip:./', 'torch', 'datasets', 'accelerate', 'evaluate', 'tqdm', 'scipy', 'scikit-learn', 'tensorboard']
      # launch_train_gpu = rh.function(fn=train,
      #                                system=gpu,
      #                                reqs=reqs,
      #                                name='train_bert_glue')
      #
      # We can pass in arguments just like we would to a function:
      # launch_train_gpu(num_epochs = 3, lr = 2e-5, seed = 42, batch_size = 16
      #                  stream_logs=True)



