


"""simple docstring"""






import inspect
from typing import Callable, List, Optional, Set, Tuple, Union

import torch
from packaging import version
from safetensors.torch import storage_ptr, storage_size
from torch import nn

from .utils import logging


_lowerCamelCase  =	[nn.LayerNorm]

_lowerCamelCase  =	logging.get_logger(__name__)

_lowerCamelCase  =	version.parse(version.parse(torch.__version__).base_version)

_lowerCamelCase  =	parsed_torch_version_base >= version.parse('2.0')
_lowerCamelCase  =	parsed_torch_version_base >= version.parse('1.12')
_lowerCamelCase  =	parsed_torch_version_base >= version.parse('1.11')
_lowerCamelCase  =	parsed_torch_version_base < version.parse('1.11')


def __lowercase  (   lowerCamelCase_	:   List[str]    ,       lowerCamelCase_	:   Optional[Any]    ,       lowerCamelCase_	:   int    ,       lowerCamelCase_	:   List[str]    ,       self	:   Union[str, Any]	):
				from torch import _softmax_backward_data

				if is_torch_less_than_1_11:
								return _softmax_backward_data(lowerCamelCase_    ,       lowerCamelCase_    ,       parent.dim    ,       self	)
				else:
								return _softmax_backward_data(lowerCamelCase_    ,       lowerCamelCase_    ,       parent.dim    ,       self.dtype	)


def __lowercase  (   lowerCamelCase_	:   nn.Linear    ,       lowerCamelCase_	:   torch.LongTensor    ,       lowerCamelCase_	:   int = 0	):
				SCREAMING_SNAKE_CASE__  				=					index.to(layer.weight.device	)
				SCREAMING_SNAKE_CASE__  				=					layer.weight.index_select(lowerCamelCase_    ,       lowerCamelCase_	).clone().detach()
				if layer.bias is not None:
								if dim == 1:
												SCREAMING_SNAKE_CASE__  				=					layer.bias.clone().detach()
								else:
												SCREAMING_SNAKE_CASE__  				=					layer.bias[index].clone().detach()
				SCREAMING_SNAKE_CASE__  				=					list(layer.weight.size()	)
				SCREAMING_SNAKE_CASE__  				=					len(lowerCamelCase_	)
				SCREAMING_SNAKE_CASE__  				=					nn.Linear(new_size[1]    ,       new_size[0]    ,       bias=layer.bias is not None	).to(layer.weight.device	)
				SCREAMING_SNAKE_CASE__  				=					False
				new_layer.weight.copy_(W.contiguous()	)
				SCREAMING_SNAKE_CASE__  				=					True
				if layer.bias is not None:
								SCREAMING_SNAKE_CASE__  				=					False
								new_layer.bias.copy_(b.contiguous()	)
								SCREAMING_SNAKE_CASE__  				=					True
				return new_layer


class       lowerCamelCase_		(      nn.Module ):





				"""simple docstring"""







				def __init__(    self							,		UpperCAmelCase__							,		UpperCAmelCase__  ):
								super().__init__()
								SCREAMING_SNAKE_CASE__  				=					nf
								SCREAMING_SNAKE_CASE__  				=					nn.Parameter(torch.empty(__A							,		__A  )  )
								SCREAMING_SNAKE_CASE__  				=					nn.Parameter(torch.zeros(__A  )  )
								nn.init.normal_(self.weight							,		std=0.02  )






				def 	lowerCAmelCase__		(    self							,		UpperCAmelCase__  ):
								SCREAMING_SNAKE_CASE__  				=					x.size()[:-1] + (self.nf,)
								SCREAMING_SNAKE_CASE__  				=					torch.addmm(self.bias							,		x.view(-1							,		x.size(-1  )  )							,		self.weight  )
								SCREAMING_SNAKE_CASE__  				=					x.view(__A  )
								return x


def __lowercase  (   lowerCamelCase_	:   ConvaD    ,       lowerCamelCase_	:   torch.LongTensor    ,       lowerCamelCase_	:   int = 1	):
				SCREAMING_SNAKE_CASE__  				=					index.to(layer.weight.device	)
				SCREAMING_SNAKE_CASE__  				=					layer.weight.index_select(lowerCamelCase_    ,       lowerCamelCase_	).clone().detach()
				if dim == 0:
								SCREAMING_SNAKE_CASE__  				=					layer.bias.clone().detach()
				else:
								SCREAMING_SNAKE_CASE__  				=					layer.bias[index].clone().detach()
				SCREAMING_SNAKE_CASE__  				=					list(layer.weight.size()	)
				SCREAMING_SNAKE_CASE__  				=					len(lowerCamelCase_	)
				SCREAMING_SNAKE_CASE__  				=					ConvaD(new_size[1]    ,       new_size[0]	).to(layer.weight.device	)
				SCREAMING_SNAKE_CASE__  				=					False
				new_layer.weight.copy_(W.contiguous()	)
				SCREAMING_SNAKE_CASE__  				=					True
				SCREAMING_SNAKE_CASE__  				=					False
				new_layer.bias.copy_(b.contiguous()	)
				SCREAMING_SNAKE_CASE__  				=					True
				return new_layer


def __lowercase  (   lowerCamelCase_	:   Union[nn.Linear, ConvaD]    ,       lowerCamelCase_	:   torch.LongTensor    ,       lowerCamelCase_	:   Optional[int] = None	):
				if isinstance(lowerCamelCase_    ,       nn.Linear	):
								return prune_linear_layer(lowerCamelCase_    ,       lowerCamelCase_    ,       dim=0 if dim is None else dim	)
				elif isinstance(lowerCamelCase_    ,       lowerCamelCase_	):
								return prune_convad_layer(lowerCamelCase_    ,       lowerCamelCase_    ,       dim=1 if dim is None else dim	)
				else:
								raise ValueError(F'''Can\'t prune layer of class {layer.__class__}'''	)


def __lowercase  (   lowerCamelCase_	:   Callable[..., torch.Tensor]    ,       lowerCamelCase_	:   int    ,       lowerCamelCase_	:   int    ,       *lowerCamelCase_	:   Tuple	):
				assert len(lowerCamelCase_	) > 0, F'''{input_tensors} has to be a tuple/list of tensors'''

				# inspect.signature exist since python 3.5 and is a python method -> no problem with backward compatibility
				SCREAMING_SNAKE_CASE__  				=					len(inspect.signature(lowerCamelCase_	).parameters	)
				if num_args_in_forward_chunk_fn != len(lowerCamelCase_	):
								raise ValueError(
								    F'''forward_chunk_fn expects {num_args_in_forward_chunk_fn} arguments, but only {len(lowerCamelCase_	)} input '''
								    "tensors are given"	)

				if chunk_size > 0:
								SCREAMING_SNAKE_CASE__  				=					input_tensors[0].shape[chunk_dim]
								for input_tensor in input_tensors:
												if input_tensor.shape[chunk_dim] != tensor_shape:
																raise ValueError(
																    F'''All input tenors have to be of the same shape: {tensor_shape}, '''
																    F'''found shape {input_tensor.shape[chunk_dim]}'''	)

								if input_tensors[0].shape[chunk_dim] % chunk_size != 0:
												raise ValueError(
												    F'''The dimension to be chunked {input_tensors[0].shape[chunk_dim]} has to be a multiple of the chunk '''
												    F'''size {chunk_size}'''	)

								SCREAMING_SNAKE_CASE__  				=					input_tensors[0].shape[chunk_dim] // chunk_size

								# chunk input tensor into tuples
								SCREAMING_SNAKE_CASE__  				=					tuple(input_tensor.chunk(lowerCamelCase_    ,       dim=lowerCamelCase_	) for input_tensor in input_tensors	)
								# apply forward fn to every tuple
								SCREAMING_SNAKE_CASE__  				=					tuple(forward_fn(*lowerCamelCase_	) for input_tensors_chunk in zip(*lowerCamelCase_	)	)
								# concatenate output at same dimension
								return torch.cat(lowerCamelCase_    ,       dim=lowerCamelCase_	)

				return forward_fn(*lowerCamelCase_	)


def __lowercase  (   lowerCamelCase_	:   List[int]    ,       lowerCamelCase_	:   int    ,       lowerCamelCase_	:   int    ,       lowerCamelCase_	:   Set[int]	):
				SCREAMING_SNAKE_CASE__  				=					torch.ones(lowerCamelCase_    ,       lowerCamelCase_	)
				SCREAMING_SNAKE_CASE__  				=					set(lowerCamelCase_	) - already_pruned_heads  # Convert to set and remove already pruned heads
				for head in heads:
								# Compute how many pruned heads are before the head and move the index accordingly
								SCREAMING_SNAKE_CASE__  				=					head - sum(1 if h < head else 0 for h in already_pruned_heads	)
								SCREAMING_SNAKE_CASE__  				=					0
				SCREAMING_SNAKE_CASE__  				=					mask.view(-1	).contiguous().eq(1	)
				SCREAMING_SNAKE_CASE__  				=					torch.arange(len(lowerCamelCase_	)	)[mask].long()
				return heads, index


def __lowercase  (   *lowerCamelCase_	:   Union[torch.Tensor, List[torch.Tensor]]    ,       lowerCamelCase_	:   Optional[str] = None	):
				return torch.meshgrid(*lowerCamelCase_    ,       indexing=lowerCamelCase_	)


def __lowercase  (   lowerCamelCase_	:   torch.Tensor	):
				return tensor.device, storage_ptr(lowerCamelCase_	), storage_size(lowerCamelCase_	)



from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


_lowerCamelCase							:  Union[str, Any]     =				logging.get_logger(__name__)

_lowerCamelCase							:  Optional[int]     =				{
    "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/config.json",
    # See all M2M100 models at https://huggingface.co/models?filter=m2m_100
}





class    __UpperCAmelCase	(			lowerCamelCase__		):
   UpperCamelCase  	=	"""m2m_100"""
   UpperCamelCase  	=	["""past_key_values"""]
   UpperCamelCase  	=	{"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}



   def __init__(				self				:	int,						__A				:	Optional[Any]=1_2_8_1_1_2,						__A				:	str=1_0_2_4,						__A				:	Tuple=1_2,						__A				:	Any=4_0_9_6,						__A				:	str=1_6,						__A				:	List[str]=1_2,						__A				:	int=4_0_9_6,						__A				:	List[Any]=1_6,						__A				:	Tuple=0.0_5,						__A				:	Tuple=0.0_5,						__A				:	List[Any]=True,						__A				:	int=True,						__A				:	str="relu",						__A				:	str=1_0_2_4,						__A				:	str=0.1,						__A				:	str=0.1,						__A				:	List[str]=0.0,						__A				:	List[str]=0.0_2,						__A				:	Any=2,						__A				:	List[str]=True,						__A				:	Tuple=1,						__A				:	Optional[Any]=0,						__A				:	Optional[Any]=2,						**__A				:	Optional[Any],						):
     UpperCAmelCase			:				str										=					vocab_size
     UpperCAmelCase			:				Optional[Any]										=					max_position_embeddings
     UpperCAmelCase			:				List[str]										=					d_model
     UpperCAmelCase			:				Any										=					encoder_ffn_dim
     UpperCAmelCase			:				Optional[int]										=					encoder_layers
     UpperCAmelCase			:				List[Any]										=					encoder_attention_heads
     UpperCAmelCase			:				str										=					decoder_ffn_dim
     UpperCAmelCase			:				Any										=					decoder_layers
     UpperCAmelCase			:				List[str]										=					decoder_attention_heads
     UpperCAmelCase			:				str										=					dropout
     UpperCAmelCase			:				int										=					attention_dropout
     UpperCAmelCase			:				Dict										=					activation_dropout
     UpperCAmelCase			:				Tuple										=					activation_function
     UpperCAmelCase			:				Any										=					init_std
     UpperCAmelCase			:				Union[str, Any]										=					encoder_layerdrop
     UpperCAmelCase			:				List[Any]										=					decoder_layerdrop
     UpperCAmelCase			:				Optional[Any]										=					use_cache
     UpperCAmelCase			:				str										=					encoder_layers
     UpperCAmelCase			:				List[Any]										=					scale_embedding  # scale factor will be sqrt(d_model) if True

     super().__init__(
         pad_token_id=__A,						bos_token_id=__A,						eos_token_id=__A,						is_encoder_decoder=__A,						decoder_start_token_id=__A,						**__A,						)






class    __UpperCAmelCase	(			lowerCamelCase__		):



   @property
   def 			__magic_name__       (				self				:	List[str]		):
     UpperCAmelCase			:				List[Any]										=					OrderedDict(
         [
             ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
             ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
         ]		)

     if self.use_past:
       UpperCAmelCase			:				List[str]										=					{0: '''batch'''}
       UpperCAmelCase			:				Any										=					{0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
     else:
       UpperCAmelCase			:				Any										=					{0: '''batch''', 1: '''decoder_sequence'''}
       UpperCAmelCase			:				List[Any]										=					{0: '''batch''', 1: '''decoder_sequence'''}

     if self.use_past:
       self.fill_with_past_key_values_(__A,						direction='''inputs'''		)
     return common_inputs



   def 			__magic_name__       (				self				:	Dict,						__A				:	PreTrainedTokenizer,						__A				:	int = -1,						__A				:	int = -1,						__A				:	bool = False,						__A				:	Optional[TensorType] = None,						):
     # Copied from OnnxConfig.generate_dummy_inputs
     # Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
     # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
     UpperCAmelCase			:				List[str]										=					compute_effective_axis_dimension(
         __A,						fixed_dimension=OnnxConfig.default_fixed_batch,						num_token_to_add=0		)

     # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
     UpperCAmelCase			:				Optional[int]										=					tokenizer.num_special_tokens_to_add(__A		)
     UpperCAmelCase			:				int										=					compute_effective_axis_dimension(
         __A,						fixed_dimension=OnnxConfig.default_fixed_sequence,						num_token_to_add=__A		)

     # Generate dummy inputs according to compute batch and sequence
     UpperCAmelCase			:				List[str]										=					[''' '''.join([tokenizer.unk_token]		) * seq_length] * batch_size
     UpperCAmelCase			:				Tuple										=					dict(tokenizer(__A,						return_tensors=__A		)		)
     return common_inputs



   def 			__magic_name__       (				self				:	Any,						__A				:	PreTrainedTokenizer,						__A				:	int = -1,						__A				:	int = -1,						__A				:	bool = False,						__A				:	Optional[TensorType] = None,						):
     UpperCAmelCase			:				int										=					self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
         __A,						__A,						__A,						__A,						__A		)

     # Generate decoder inputs
     UpperCAmelCase			:				Union[str, Any]										=					seq_length if not self.use_past else 1
     UpperCAmelCase			:				Optional[int]										=					self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
         __A,						__A,						__A,						__A,						__A		)
     UpperCAmelCase			:				Dict										=					{F'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
     UpperCAmelCase			:				Optional[int]										=					dict(**__A,						**__A		)

     if self.use_past:
       if not is_torch_available():
         raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''		)
       else:
         import torch
       UpperCAmelCase					,		UpperCAmelCase			:				List[str]										=					common_inputs['''input_ids'''].shape
       UpperCAmelCase			:				Dict										=					common_inputs['''decoder_input_ids'''].shape[1]
       UpperCAmelCase					,		UpperCAmelCase			:				Dict										=					self.num_attention_heads
       UpperCAmelCase			:				Union[str, Any]										=					(
           batch,
           num_encoder_attention_heads,
           encoder_seq_length,
           self._config.hidden_size // num_encoder_attention_heads,
       )
       UpperCAmelCase			:				str										=					decoder_seq_length + 3
       UpperCAmelCase			:				Union[str, Any]										=					(
           batch,
           num_decoder_attention_heads,
           decoder_past_length,
           self._config.hidden_size // num_decoder_attention_heads,
       )

       UpperCAmelCase			:				Union[str, Any]										=					torch.cat(
           [common_inputs['''decoder_attention_mask'''], torch.ones(__A,						__A		)],						dim=1		)

       UpperCAmelCase			:				Any										=					[]
       # If the number of encoder and decoder layers are present in the model configuration, both are considered
       UpperCAmelCase					,		UpperCAmelCase			:				str										=					self.num_layers
       UpperCAmelCase			:				Any										=					min(__A,						__A		)
       UpperCAmelCase			:				Dict										=					max(__A,						__A		) - min_num_layers
       UpperCAmelCase			:				Tuple										=					'''encoder''' if num_encoder_layers > num_decoder_layers else '''decoder'''

       for _ in range(__A		):
         common_inputs["past_key_values"].append(
             (
                 torch.zeros(__A		),
                 torch.zeros(__A		),
                 torch.zeros(__A		),
                 torch.zeros(__A		),
             )		)
       # TODO: test this.
       UpperCAmelCase			:				List[str]										=					encoder_shape if remaining_side_name == '''encoder''' else decoder_shape
       for _ in range(__A,						__A		):
         common_inputs["past_key_values"].append((torch.zeros(__A		), torch.zeros(__A		))		)
     return common_inputs





   UpperCamelCase  	=	_generate_dummy_inputs_for_default_and_seqaseq_lm





