



'''simple docstring'''







from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_UpperCamelCase  						=			logging.get_logger(__name__)

_UpperCamelCase  						=			{
    """mnaylor/mega-base-wikitext""": """https://huggingface.co/mnaylor/mega-base-wikitext/resolve/main/config.json""",
}

class      lowerCamelCase__     (     _A				):






	'''simple docstring'''

	A__		=	'''mega'''




	def __init__(  self		:  Dict   ,		__A		:  Any=3_0522   ,		__A		:  Dict=128   ,		__A		:  Union[str, Any]=4   ,		__A		:  Optional[Any]=256   ,		__A		:  List[str]=16   ,		__A		:  Any=True   ,		__A		:  str=64   ,		__A		:  List[str]=False   ,		__A		:  Union[str, Any]=-1   ,		__A		:  str=None   ,		__A		:  Optional[int]=True   ,		__A		:  Tuple="scalenorm"   ,		__A		:  Dict=True   ,		__A		:  Union[str, Any]="silu"   ,		__A		:  Tuple="softmax"   ,		__A		:  int=0.1   ,		__A		:  Optional[int]=0.1   ,		__A		:  Optional[Any]=0.1   ,		__A		:  Union[str, Any]=False   ,		__A		:  Tuple=True   ,		__A		:  Any=256   ,		__A		:  List[Any]=True   ,		__A		:  List[str]=0.1   ,		__A		:  Optional[Any]=2048   ,		__A		:  Tuple=False   ,		__A		:  int=2   ,		__A		:  Any=0.0_2   ,		__A		:  int=0.2   ,		__A		:  Union[str, Any]=0.0_2   ,		__A		:  Union[str, Any]=1.0   ,		__A		:  Dict=1   ,		__A		:  Optional[Any]=0   ,		__A		:  str=2   ,		__A		:  int="rotary"   ,		__A		:  int=None   ,		__A		:  Any=True   ,		__A		:  Tuple=True   ,		**__A		:  Dict   ,		)     ->			List[str]:




								'''simple docstring'''





								super().__init__(pad_token_id=__A   ,		bos_token_id=__A   ,		eos_token_id=__A   ,		**__A					)

								lowerCAmelCase__  =   vocab_size
								lowerCAmelCase__  =   hidden_size
								lowerCAmelCase__  =   num_hidden_layers
								lowerCAmelCase__  =   activation
								lowerCAmelCase__  =   attention_activation
								lowerCAmelCase__  =   intermediate_size
								lowerCAmelCase__  =   ema_projection_size
								lowerCAmelCase__  =   bidirectional
								lowerCAmelCase__  =   shared_representation_size
								lowerCAmelCase__  =   use_chunking
								lowerCAmelCase__  =   chunk_size
								lowerCAmelCase__  =   truncation
								lowerCAmelCase__  =   normalize_before_mega
								lowerCAmelCase__  =   normalization_type
								lowerCAmelCase__  =   norm_affine
								lowerCAmelCase__  =   dropout_prob
								lowerCAmelCase__  =   hidden_dropout_prob
								lowerCAmelCase__  =   attention_probs_dropout_prob
								lowerCAmelCase__  =   use_feature_dropout
								lowerCAmelCase__  =   use_normalized_ffn
								lowerCAmelCase__  =   nffn_hidden_size
								lowerCAmelCase__  =   normalize_before_ffn
								lowerCAmelCase__  =   nffn_activation_dropout_prob
								lowerCAmelCase__  =   max_positions
								lowerCAmelCase__  =   add_token_type_embeddings
								lowerCAmelCase__  =   type_vocab_size
								lowerCAmelCase__  =   initializer_range
								lowerCAmelCase__  =   ema_delta_alpha_range
								lowerCAmelCase__  =   ema_beta_range
								lowerCAmelCase__  =   ema_gamma_omega_range
								lowerCAmelCase__  =   relative_positional_bias
								lowerCAmelCase__  =   use_cache
								lowerCAmelCase__  =   classifier_dropout
								lowerCAmelCase__  =   add_lm_hidden_dense_layer
								lowerCAmelCase__  =   1  # not used but required by Hugging Face





class      lowerCamelCase__     (     _A				):






	'''simple docstring'''

	@property
	def      lowercase__							(  self		:  List[Any]					)     ->			Mapping[str, Mapping[int, str]]:




								'''simple docstring'''





								if self.task == "multiple-choice":
															lowerCAmelCase__  =   {0: """batch""", 1: """choice""", 2: """sequence"""}
								else:
															lowerCAmelCase__  =   {0: """batch""", 1: """sequence"""}
								return OrderedDict(
								    [
								        ("""input_ids""", dynamic_axis),
								        ("""attention_mask""", dynamic_axis),
								    ]					)










'''simple docstring'''







from functools import partial
from typing import Any, Callable, Dict, List, Type, TypeVar, Union, overload

import torch
import torch.nn as nn
import torch.types
def 			_lowerCAmelCase(      UpperCAmelCase_      :	torch.Tensor					, UpperCAmelCase_      :	torch.Tensor					, UpperCAmelCase_      :	bool						) ->							torch.Tensor:
							# The first operation in a checkpoint can't be in-place, but it's
							# nice to have in-place addition during inference. Thus...
							if not inplace:
														lowerCAmelCase__  =   ma + ma
							else:
														ma += ma

							return ma
def 			_lowerCAmelCase(      UpperCAmelCase_      :	torch.Tensor					, UpperCAmelCase_      :	List[int]						) ->							torch.Tensor:
							lowerCAmelCase__  =   -1 * len(UpperCAmelCase_						)
							lowerCAmelCase__  =   list(range(len(tensor.shape[:zero_index]						)						)						)
							return tensor.permute(first_inds + [zero_index + i for i in inds]						)
def 			_lowerCAmelCase(      UpperCAmelCase_      :	torch.Tensor					, UpperCAmelCase_      :	int						) ->							torch.Tensor:
							return t.reshape(t.shape[:-no_dims] + (-1,)						)
def 			_lowerCAmelCase(      UpperCAmelCase_      :	torch.Tensor					, UpperCAmelCase_      :	torch.Tensor					, UpperCAmelCase_      :	int					, UpperCAmelCase_      :	float = 1E-4						) ->							torch.Tensor:
							lowerCAmelCase__  =   mask.expand(*value.shape						)
							return torch.sum(mask * value					, dim=UpperCAmelCase_						) / (eps + torch.sum(UpperCAmelCase_					, dim=UpperCAmelCase_						))
def 			_lowerCAmelCase(      UpperCAmelCase_      :	torch.Tensor					, UpperCAmelCase_      :	torch.types.Number = 2.31_25					, UpperCAmelCase_      :	torch.types.Number = 21.68_75					, UpperCAmelCase_      :	int = 64						) ->							torch.Tensor:
							lowerCAmelCase__  =   torch.linspace(UpperCAmelCase_					, UpperCAmelCase_					, no_bins - 1					, device=pts.device						)
							lowerCAmelCase__  =   torch.sqrt(torch.sum((pts.unsqueeze(-2						) - pts.unsqueeze(-3						)) ** 2					, dim=-1						)						)
							return torch.bucketize(UpperCAmelCase_					, UpperCAmelCase_						)
def 			_lowerCAmelCase(      UpperCAmelCase_      :	Callable[[list], Any]					, UpperCAmelCase_      :	List[dict]						) ->							dict:
							lowerCAmelCase__  =   dicts[0]
							lowerCAmelCase__  =   {}
							for k, v in first.items():
														lowerCAmelCase__  =   [d[k] for d in dicts]
														if isinstance(UpperCAmelCase_					, UpperCAmelCase_						):
																					lowerCAmelCase__  =   dict_multimap(UpperCAmelCase_					, UpperCAmelCase_						)
														else:
																					lowerCAmelCase__  =   fn(UpperCAmelCase_						)

							return new_dict
def 			_lowerCAmelCase(      UpperCAmelCase_      :	torch.Tensor					, UpperCAmelCase_      :	torch.Tensor						) ->							torch.Tensor:
							lowerCAmelCase__  =   v_bins.view(((1,) * len(x.shape						)) + (len(UpperCAmelCase_						),)						)
							lowerCAmelCase__  =   x[..., None] - reshaped_bins
							lowerCAmelCase__  =   torch.argmin(torch.abs(UpperCAmelCase_						)					, dim=-1						)
							return nn.functional.one_hot(UpperCAmelCase_					, num_classes=len(UpperCAmelCase_						)						).float()
def 			_lowerCAmelCase(      UpperCAmelCase_      :	torch.Tensor					, UpperCAmelCase_      :	torch.Tensor					, UpperCAmelCase_      :	int = 0					, UpperCAmelCase_      :	int = 0						) ->							torch.Tensor:
							lowerCAmelCase__  =   []
							for i, s in enumerate(data.shape[:no_batch_dims]						):
														lowerCAmelCase__  =   torch.arange(UpperCAmelCase_						)
														lowerCAmelCase__  =   r.view(*(*((1,) * i), -1, *((1,) * (len(inds.shape						) - i - 1)))						)
														ranges.append(UpperCAmelCase_						)

							lowerCAmelCase__  =   [slice(UpperCAmelCase_						) for _ in range(len(data.shape						) - no_batch_dims						)]
							lowerCAmelCase__  =   inds
							ranges.extend(UpperCAmelCase_						)
							# Matt note: Editing this to get around the behaviour of using a list as an array index changing
							# in recent Numpy versions
							return data[tuple(UpperCAmelCase_						)]


_UpperCamelCase  						=			TypeVar("""T""")
def 			_lowerCAmelCase(      UpperCAmelCase_      :	Callable[[T], Any]					, UpperCAmelCase_      :	Dict[Any, Union[dict, list, tuple, T]]					, UpperCAmelCase_      :	Type[T]						) ->							Dict[Any, Union[dict, list, tuple, Any]]:
							lowerCAmelCase__  =   {}
							for k, v in dic.items():
														if isinstance(UpperCAmelCase_					, UpperCAmelCase_						):
																					lowerCAmelCase__  =   dict_map(UpperCAmelCase_					, UpperCAmelCase_					, UpperCAmelCase_						)
														else:
																					lowerCAmelCase__  =   tree_map(UpperCAmelCase_					, UpperCAmelCase_					, UpperCAmelCase_						)

							return new_dict
@overload
def 			_lowerCAmelCase(      UpperCAmelCase_      :	Callable[[T], Any]					, UpperCAmelCase_      :	T					, UpperCAmelCase_      :	Type[T]						) ->							Any:
							...
@overload
def 			_lowerCAmelCase(      UpperCAmelCase_      :	Callable[[T], Any]					, UpperCAmelCase_      :	dict					, UpperCAmelCase_      :	Type[T]						) ->							dict:
							...
@overload
def 			_lowerCAmelCase(      UpperCAmelCase_      :	Callable[[T], Any]					, UpperCAmelCase_      :	list					, UpperCAmelCase_      :	Type[T]						) ->							list:
							...
@overload
def 			_lowerCAmelCase(      UpperCAmelCase_      :	Callable[[T], Any]					, UpperCAmelCase_      :	tuple					, UpperCAmelCase_      :	Type[T]						) ->							tuple:
							...
def 			_lowerCAmelCase(      UpperCAmelCase_      :	str					, UpperCAmelCase_      :	Dict					, UpperCAmelCase_      :	Union[str, Any]						) ->							List[str]:
							if isinstance(UpperCAmelCase_					, UpperCAmelCase_						):
														return dict_map(UpperCAmelCase_					, UpperCAmelCase_					, UpperCAmelCase_						)
							elif isinstance(UpperCAmelCase_					, UpperCAmelCase_						):
														return [tree_map(UpperCAmelCase_					, UpperCAmelCase_					, UpperCAmelCase_						) for x in tree]
							elif isinstance(UpperCAmelCase_					, UpperCAmelCase_						):
														return tuple(tree_map(UpperCAmelCase_					, UpperCAmelCase_					, UpperCAmelCase_						) for x in tree						)
							elif isinstance(UpperCAmelCase_					, UpperCAmelCase_						):
														return fn(UpperCAmelCase_						)
							else:
														print(type(UpperCAmelCase_						)						)
														raise ValueError("""Not supported"""						)






_UpperCamelCase  						=			partial(tree_map, leaf_type=torch.Tensor)





