




import json
import os
from dataclasses import dataclass
from enum import Enum
from typing import List, Optional, Union

import yaml

from ...utils import ComputeEnvironment, DistributedType, SageMakerDistributedType
from ...utils.constants import SAGEMAKER_PYTHON_VERSION, SAGEMAKER_PYTORCH_VERSION, SAGEMAKER_TRANSFORMERS_VERSION


lowercase_      :							Optional[int]    							=					os.path.expanduser(
    os.getenv('HF_HOME', os.path.join(os.getenv('XDG_CACHE_HOME', '~/.cache'), 'huggingface'))
)
lowercase_      :							int    							=					os.path.join(hf_cache_home, 'accelerate')
lowercase_      :							Union[str, Any]    							=					os.path.join(cache_dir, 'default_config.yaml')
lowercase_      :							Optional[Any]    							=					os.path.join(cache_dir, 'default_config.yaml')

# For backward compatibility: the default config is the json one if it's the only existing file.
if os.path.isfile(default_yaml_config_file) or not os.path.isfile(default_json_config_file):
				lowercase_      :							Dict    							=					default_yaml_config_file
else:
				lowercase_      :							Optional[Any]    							=					default_json_config_file


def 				__SCREAMING_SNAKE_CASE    (     snake_case_		):

						'''simple docstring'''


						if config_file is not None:
												if not os.path.isfile(snake_case_		):
																		raise FileNotFoundError(
																		    f"""The passed configuration file `{config_file}` does not exist. """
																		    "Please pass an existing file to `accelerate launch`, or use the the default one "
																		    "created through `accelerate config` and run `accelerate launch` "
																		    "without the `--config_file` argument."		)
						else:
												_UpperCAmelCase						=	default_config_file
						with open(snake_case_      ,       "r"      ,       encoding="utf-8"		) as f:
												if config_file.endswith(".json"		):
																		if (
																		    json.load(snake_case_		).get("compute_environment"      ,       ComputeEnvironment.LOCAL_MACHINE		)
																		    == ComputeEnvironment.LOCAL_MACHINE
																		):
																								_UpperCAmelCase						=	ClusterConfig
																		else:
																								_UpperCAmelCase						=	SageMakerConfig
																		return config_class.from_json_file(json_file=snake_case_		)
												else:
																		if (
																		    yaml.safe_load(snake_case_		).get("compute_environment"      ,       ComputeEnvironment.LOCAL_MACHINE		)
																		    == ComputeEnvironment.LOCAL_MACHINE
																		):
																								_UpperCAmelCase						=	ClusterConfig
																		else:
																								_UpperCAmelCase						=	SageMakerConfig
																		return config_class.from_yaml_file(yaml_file=snake_case_		)






@dataclass
class        __lowerCAmelCase					:
			snake_case_ :					ComputeEnvironment
			snake_case_ :					Union[DistributedType, SageMakerDistributedType]
			snake_case_ :					str
			snake_case_ :					bool







			def UpperCamelCase     (			self       :     str ):




									"""simple docstring"""


									_UpperCAmelCase						=	self.__dict__
									# For serialization, it's best to convert Enums to strings (or their underlying value type).
									for key, value in result.items():
															if isinstance(_snake_case       ,       _snake_case ):
																					_UpperCAmelCase						=	value.value
															if isinstance(_snake_case       ,       _snake_case ) and not bool(_snake_case ):
																					_UpperCAmelCase						=	None
									_UpperCAmelCase						=	{k: v for k, v in result.items() if v is not None}
									return result







			@classmethod
			def UpperCamelCase     (			cls       :     Union[str, Any]       ,       snake_case__       :     Any=None ):




									"""simple docstring"""


									_UpperCAmelCase						=	default_json_config_file if json_file is None else json_file
									with open(_snake_case       ,       "r"       ,       encoding="utf-8" ) as f:
															_UpperCAmelCase						=	json.load(_snake_case )
									if "compute_environment" not in config_dict:
															_UpperCAmelCase						=	ComputeEnvironment.LOCAL_MACHINE
									if "mixed_precision" not in config_dict:
															_UpperCAmelCase						=	'fp16' if ('fp16' in config_dict and config_dict['fp16']) else None
									if "fp16" in config_dict:  # Convert the config to the new format.
															del config_dict["fp16"]
									if "dynamo_backend" in config_dict:  # Convert the config to the new format.
															_UpperCAmelCase						=	config_dict.pop("dynamo_backend" )
															_UpperCAmelCase						=	{} if dynamo_backend == 'NO' else {'dynamo_backend': dynamo_backend}
									if "use_cpu" not in config_dict:
															_UpperCAmelCase						=	False
									return cls(**_snake_case )







			def UpperCamelCase     (			self       :     Optional[int]       ,       snake_case__       :     Optional[Any] ):




									"""simple docstring"""


									with open(_snake_case       ,       "w"       ,       encoding="utf-8" ) as f:
															_UpperCAmelCase						=	json.dumps(self.to_dict()       ,       indent=2       ,       sort_keys=_snake_case ) + '\n'
															f.write(_snake_case )







			@classmethod
			def UpperCamelCase     (			cls       :     Optional[Any]       ,       snake_case__       :     Union[str, Any]=None ):




									"""simple docstring"""


									_UpperCAmelCase						=	default_yaml_config_file if yaml_file is None else yaml_file
									with open(_snake_case       ,       "r"       ,       encoding="utf-8" ) as f:
															_UpperCAmelCase						=	yaml.safe_load(_snake_case )
									if "compute_environment" not in config_dict:
															_UpperCAmelCase						=	ComputeEnvironment.LOCAL_MACHINE

									if "mixed_precision" not in config_dict:
															_UpperCAmelCase						=	'fp16' if ('fp16' in config_dict and config_dict['fp16']) else None
									if isinstance(config_dict["mixed_precision"]       ,       _snake_case ) and not config_dict["mixed_precision"]:
															_UpperCAmelCase						=	'no'
									if "fp16" in config_dict:  # Convert the config to the new format.
															del config_dict["fp16"]
									if "dynamo_backend" in config_dict:  # Convert the config to the new format.
															_UpperCAmelCase						=	config_dict.pop("dynamo_backend" )
															_UpperCAmelCase						=	{} if dynamo_backend == 'NO' else {'dynamo_backend': dynamo_backend}
									if "use_cpu" not in config_dict:
															_UpperCAmelCase						=	False
									return cls(**_snake_case )







			def UpperCamelCase     (			self       :     str       ,       snake_case__       :     str ):




									"""simple docstring"""


									with open(_snake_case       ,       "w"       ,       encoding="utf-8" ) as f:
															yaml.safe_dump(self.to_dict()       ,       _snake_case )







			def UpperCamelCase     (			self       :     Union[str, Any] ):




									"""simple docstring"""


									if isinstance(self.compute_environment       ,       _snake_case ):
															_UpperCAmelCase						=	ComputeEnvironment(self.compute_environment )
									if isinstance(self.distributed_type       ,       _snake_case ):
															if self.compute_environment == ComputeEnvironment.AMAZON_SAGEMAKER:
																					_UpperCAmelCase						=	SageMakerDistributedType(self.distributed_type )
															else:
																					_UpperCAmelCase						=	DistributedType(self.distributed_type )
									if self.dynamo_config is None:
															_UpperCAmelCase						=	{}






@dataclass
class        __lowerCAmelCase					(			a_					):
			snake_case_ :					int
			snake_case_ :					int						=	0
			snake_case_ :					int						=	1
			snake_case_ :					Optional[str]						=	None
			snake_case_ :					Optional[str]						=	None
			snake_case_ :					Optional[int]						=	None
			snake_case_ :					Optional[str]						=	"static"
			snake_case_ :					Optional[bool]						=	False
			snake_case_ :					str						=	"main"

			# args for deepspeed_plugin
			snake_case_ :					dict						=	None
			# args for fsdp
			snake_case_ :					dict						=	None
			# args for megatron_lm
			snake_case_ :					dict						=	None
			# args for ipex
			snake_case_ :					dict						=	None
			# args for TPU
			snake_case_ :					bool						=	False

			# args for TPU pods
			snake_case_ :					str						=	None
			snake_case_ :					str						=	None
			snake_case_ :					bool						=	False
			snake_case_ :					bool						=	False
			snake_case_ :					str						=	None
			snake_case_ :					List[str]						=	None
			snake_case_ :					List[str]						=	None
			snake_case_ :					List[str]						=	None

			# args for dynamo
			snake_case_ :					dict						=	None







			def UpperCamelCase     (			self       :     List[str] ):




									"""simple docstring"""


									if self.deepspeed_config is None:
															_UpperCAmelCase						=	{}
									if self.fsdp_config is None:
															_UpperCAmelCase						=	{}
									if self.megatron_lm_config is None:
															_UpperCAmelCase						=	{}
									if self.ipex_config is None:
															_UpperCAmelCase						=	{}
									return super().__post_init__()






@dataclass
class        __lowerCAmelCase					(			a_					):
			snake_case_ :					str
			snake_case_ :					str
			snake_case_ :					Optional[str]						=	None
			snake_case_ :					Optional[str]						=	None
			snake_case_ :					str						=	"us-east-1"
			snake_case_ :					int						=	1
			snake_case_ :					str						=	"all"
			snake_case_ :					str						=	F"""accelerate-sagemaker-{num_machines}"""
			snake_case_ :					str						=	SAGEMAKER_PYTORCH_VERSION
			snake_case_ :					str						=	SAGEMAKER_TRANSFORMERS_VERSION
			snake_case_ :					str						=	SAGEMAKER_PYTHON_VERSION
			snake_case_ :					str						=	None
			snake_case_ :					str						=	None
			snake_case_ :					dict						=	None
			snake_case_ :					dict						=	None







'''simple docstring'''


import inspect
from typing import Callable, List, Optional, Set, Tuple, Union

import torch
from packaging import version
from safetensors.torch import storage_ptr, storage_size
from torch import nn

from .utils import logging


lowerCAmelCase__      				=			[nn.LayerNorm]

lowerCAmelCase__      				=			logging.get_logger(__name__)

lowerCAmelCase__      				=			version.parse(version.parse(torch.__version__).base_version)

lowerCAmelCase__      				=			parsed_torch_version_base >= version.parse("2.0")
lowerCAmelCase__      				=			parsed_torch_version_base >= version.parse("1.12")
lowerCAmelCase__      				=			parsed_torch_version_base >= version.parse("1.11")
lowerCAmelCase__      				=			parsed_torch_version_base < version.parse("1.11")







def     SCREAMING_SNAKE_CASE(	UpperCamelCase ,UpperCamelCase ,UpperCamelCase ,UpperCamelCase ,self	)     ->     List[Any]:
	from torch import _softmax_backward_data

	if is_torch_less_than_1_11:
		return _softmax_backward_data(UpperCamelCase ,UpperCamelCase ,parent.dim ,self	)
	else:
		return _softmax_backward_data(UpperCamelCase ,UpperCamelCase ,parent.dim ,self.dtype	)







def     SCREAMING_SNAKE_CASE(	UpperCamelCase ,UpperCamelCase ,UpperCamelCase = 0	)     ->     nn.Linear:
	UpperCAmelCase_		:   Tuple													=       index.to(layer.weight.device	)
	UpperCAmelCase_		:   Tuple													=       layer.weight.index_select(UpperCamelCase ,UpperCamelCase	).clone().detach()
	if layer.bias is not None:
		if dim == 1:
			UpperCAmelCase_		:   List[Any]													=       layer.bias.clone().detach()
		else:
			UpperCAmelCase_		:   Tuple													=       layer.bias[index].clone().detach()
	UpperCAmelCase_		:   Optional[Any]													=       list(layer.weight.size()	)
	UpperCAmelCase_		:   int													=       len(UpperCamelCase	)
	UpperCAmelCase_		:   str													=       nn.Linear(new_size[1] ,new_size[0] ,bias=layer.bias is not None	).to(layer.weight.device	)
	UpperCAmelCase_		:   str													=       False
	new_layer.weight.copy_(W.contiguous()	)
	UpperCAmelCase_		:   Tuple													=       True
	if layer.bias is not None:
		UpperCAmelCase_		:   Union[str, Any]													=       False
		new_layer.bias.copy_(b.contiguous()	)
		UpperCAmelCase_		:   Any													=       True
	return new_layer



class        lowercase    (	nn.Module			):


							def __init__(      self     ,      _snake_case     ,      _snake_case)				->					int:
								super().__init__()
								UpperCAmelCase_		:   List[Any]													=       nf
								UpperCAmelCase_		:   List[Any]													=       nn.Parameter(torch.empty(_snake_case     ,      _snake_case))
								UpperCAmelCase_		:   List[Any]													=       nn.Parameter(torch.zeros(_snake_case))
								nn.init.normal_(self.weight     ,      std=0.02)





							def     _snake_case				(      self     ,      _snake_case)				->					Optional[int]:
								UpperCAmelCase_		:   Dict													=       x.size()[:-1] + (self.nf,)
								UpperCAmelCase_		:   Optional[Any]													=       torch.addmm(self.bias     ,      x.view(-1     ,      x.size(-1))     ,      self.weight)
								UpperCAmelCase_		:   List[str]													=       x.view(_snake_case)
								return x







def     SCREAMING_SNAKE_CASE(	UpperCamelCase ,UpperCamelCase ,UpperCamelCase = 1	)     ->     ConvaD:
	UpperCAmelCase_		:   List[str]													=       index.to(layer.weight.device	)
	UpperCAmelCase_		:   Optional[Any]													=       layer.weight.index_select(UpperCamelCase ,UpperCamelCase	).clone().detach()
	if dim == 0:
		UpperCAmelCase_		:   Tuple													=       layer.bias.clone().detach()
	else:
		UpperCAmelCase_		:   str													=       layer.bias[index].clone().detach()
	UpperCAmelCase_		:   str													=       list(layer.weight.size()	)
	UpperCAmelCase_		:   List[str]													=       len(UpperCamelCase	)
	UpperCAmelCase_		:   List[Any]													=       ConvaD(new_size[1] ,new_size[0]	).to(layer.weight.device	)
	UpperCAmelCase_		:   Tuple													=       False
	new_layer.weight.copy_(W.contiguous()	)
	UpperCAmelCase_		:   Optional[Any]													=       True
	UpperCAmelCase_		:   Any													=       False
	new_layer.bias.copy_(b.contiguous()	)
	UpperCAmelCase_		:   Union[str, Any]													=       True
	return new_layer







def     SCREAMING_SNAKE_CASE(	UpperCamelCase ,UpperCamelCase ,UpperCamelCase = None	)     ->     Union[nn.Linear, ConvaD]:
	if isinstance(UpperCamelCase ,nn.Linear	):
		return prune_linear_layer(UpperCamelCase ,UpperCamelCase ,dim=0 if dim is None else dim	)
	elif isinstance(UpperCamelCase ,UpperCamelCase	):
		return prune_convad_layer(UpperCamelCase ,UpperCamelCase ,dim=1 if dim is None else dim	)
	else:
		raise ValueError(f"""Can't prune layer of class {layer.__class__}"""	)







def     SCREAMING_SNAKE_CASE(	UpperCamelCase ,UpperCamelCase ,UpperCamelCase ,*UpperCamelCase	)     ->     torch.Tensor:
	assert len(UpperCamelCase	) > 0, f"""{input_tensors} has to be a tuple/list of tensors"""

	# inspect.signature exist since python 3.5 and is a python method -> no problem with backward compatibility
	UpperCAmelCase_		:   str													=       len(inspect.signature(UpperCamelCase	).parameters	)
	if num_args_in_forward_chunk_fn != len(UpperCamelCase	):
		raise ValueError(
		    f"""forward_chunk_fn expects {num_args_in_forward_chunk_fn} arguments, but only {len(UpperCamelCase	)} input """
		    'tensors are given'	)

	if chunk_size > 0:
		UpperCAmelCase_		:   Optional[int]													=       input_tensors[0].shape[chunk_dim]
		for input_tensor in input_tensors:
			if input_tensor.shape[chunk_dim] != tensor_shape:
				raise ValueError(
				    f"""All input tenors have to be of the same shape: {tensor_shape}, """
				    f"""found shape {input_tensor.shape[chunk_dim]}"""	)

		if input_tensors[0].shape[chunk_dim] % chunk_size != 0:
			raise ValueError(
			    f"""The dimension to be chunked {input_tensors[0].shape[chunk_dim]} has to be a multiple of the chunk """
			    f"""size {chunk_size}"""	)

		UpperCAmelCase_		:   Any													=       input_tensors[0].shape[chunk_dim] // chunk_size

		# chunk input tensor into tuples
		UpperCAmelCase_		:   int													=       tuple(input_tensor.chunk(UpperCamelCase ,dim=UpperCamelCase	) for input_tensor in input_tensors	)
		# apply forward fn to every tuple
		UpperCAmelCase_		:   List[str]													=       tuple(forward_fn(*UpperCamelCase	) for input_tensors_chunk in zip(*UpperCamelCase	)	)
		# concatenate output at same dimension
		return torch.cat(UpperCamelCase ,dim=UpperCamelCase	)

	return forward_fn(*UpperCamelCase	)







def     SCREAMING_SNAKE_CASE(	UpperCamelCase ,UpperCamelCase ,UpperCamelCase ,UpperCamelCase	)     ->     Tuple[Set[int], torch.LongTensor]:
	UpperCAmelCase_		:   List[Any]													=       torch.ones(UpperCamelCase ,UpperCamelCase	)
	UpperCAmelCase_		:   Optional[int]													=       set(UpperCamelCase	) - already_pruned_heads  # Convert to set and remove already pruned heads
	for head in heads:
		# Compute how many pruned heads are before the head and move the index accordingly
		UpperCAmelCase_		:   Any													=       head - sum(1 if h < head else 0 for h in already_pruned_heads	)
		UpperCAmelCase_		:   List[Any]													=       0
	UpperCAmelCase_		:   str													=       mask.view(-1	).contiguous().eq(1	)
	UpperCAmelCase_		:   torch.LongTensor													=       torch.arange(len(UpperCamelCase	)	)[mask].long()
	return heads, index







def     SCREAMING_SNAKE_CASE(	*UpperCamelCase ,UpperCamelCase = None	)     ->     Tuple[torch.Tensor, ...]:
	return torch.meshgrid(*UpperCamelCase ,indexing=UpperCamelCase	)







def     SCREAMING_SNAKE_CASE(	UpperCamelCase	)     ->     Tuple[torch.device, int, int]:
	return tensor.device, storage_ptr(UpperCamelCase	), storage_size(UpperCamelCase	)






