"""simple docstring"""




from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCAmelCase       :    List[str]									=			logging.get_logger(__name__)

UpperCAmelCase       :    List[Any]									=			{
    "mnaylor/mega-base-wikitext": "https://huggingface.co/mnaylor/mega-base-wikitext/resolve/main/config.json",
}

class        __SCREAMING_SNAKE_CASE     (   _lowerCAmelCase							):
				UpperCAmelCase	=      '''mega'''







				def __init__(    self    :Dict      ,__UpperCAmelCase    :str=3_05_22      ,__UpperCAmelCase    :Any=1_28      ,__UpperCAmelCase    :Optional[Any]=4      ,__UpperCAmelCase    :int=2_56      ,__UpperCAmelCase    :Optional[int]=16      ,__UpperCAmelCase    :Optional[int]=True      ,__UpperCAmelCase    :Tuple=64      ,__UpperCAmelCase    :Any=False      ,__UpperCAmelCase    :List[str]=-1      ,__UpperCAmelCase    :Any=None      ,__UpperCAmelCase    :Tuple=True      ,__UpperCAmelCase    :str="scalenorm"      ,__UpperCAmelCase    :List[Any]=True      ,__UpperCAmelCase    :Tuple="silu"      ,__UpperCAmelCase    :Tuple="softmax"      ,__UpperCAmelCase    :Optional[int]=0.1      ,__UpperCAmelCase    :str=0.1      ,__UpperCAmelCase    :Dict=0.1      ,__UpperCAmelCase    :str=False      ,__UpperCAmelCase    :Optional[Any]=True      ,__UpperCAmelCase    :Union[str, Any]=2_56      ,__UpperCAmelCase    :Optional[int]=True      ,__UpperCAmelCase    :Any=0.1      ,__UpperCAmelCase    :Union[str, Any]=20_48      ,__UpperCAmelCase    :Union[str, Any]=False      ,__UpperCAmelCase    :Any=2      ,__UpperCAmelCase    :List[Any]=0.02      ,__UpperCAmelCase    :Tuple=0.2      ,__UpperCAmelCase    :Dict=0.02      ,__UpperCAmelCase    :Union[str, Any]=1.0      ,__UpperCAmelCase    :Dict=1      ,__UpperCAmelCase    :List[Any]=0      ,__UpperCAmelCase    :List[Any]=2      ,__UpperCAmelCase    :Union[str, Any]="rotary"      ,__UpperCAmelCase    :Optional[Any]=None      ,__UpperCAmelCase    :Optional[int]=True      ,__UpperCAmelCase    :int=True      ,**__UpperCAmelCase    :Tuple      ,)      ->					List[str]:




									"""simple docstring"""



									super().__init__(pad_token_id=__UpperCAmelCase      ,bos_token_id=__UpperCAmelCase      ,eos_token_id=__UpperCAmelCase      ,**__UpperCAmelCase  )

									lowerCamelCase__		:   Dict  				=					vocab_size
									lowerCamelCase__		:   List[str]  				=					hidden_size
									lowerCamelCase__		:   List[str]  				=					num_hidden_layers
									lowerCamelCase__		:   List[Any]  				=					activation
									lowerCamelCase__		:   int  				=					attention_activation
									lowerCamelCase__		:   str  				=					intermediate_size
									lowerCamelCase__		:   List[str]  				=					ema_projection_size
									lowerCamelCase__		:   Optional[Any]  				=					bidirectional
									lowerCamelCase__		:   int  				=					shared_representation_size
									lowerCamelCase__		:   Optional[Any]  				=					use_chunking
									lowerCamelCase__		:   Optional[Any]  				=					chunk_size
									lowerCamelCase__		:   Dict  				=					truncation
									lowerCamelCase__		:   Tuple  				=					normalize_before_mega
									lowerCamelCase__		:   int  				=					normalization_type
									lowerCamelCase__		:   List[str]  				=					norm_affine
									lowerCamelCase__		:   List[Any]  				=					dropout_prob
									lowerCamelCase__		:   str  				=					hidden_dropout_prob
									lowerCamelCase__		:   List[str]  				=					attention_probs_dropout_prob
									lowerCamelCase__		:   int  				=					use_feature_dropout
									lowerCamelCase__		:   int  				=					use_normalized_ffn
									lowerCamelCase__		:   int  				=					nffn_hidden_size
									lowerCamelCase__		:   Union[str, Any]  				=					normalize_before_ffn
									lowerCamelCase__		:   Tuple  				=					nffn_activation_dropout_prob
									lowerCamelCase__		:   Union[str, Any]  				=					max_positions
									lowerCamelCase__		:   List[str]  				=					add_token_type_embeddings
									lowerCamelCase__		:   Optional[Any]  				=					type_vocab_size
									lowerCamelCase__		:   Dict  				=					initializer_range
									lowerCamelCase__		:   str  				=					ema_delta_alpha_range
									lowerCamelCase__		:   Optional[Any]  				=					ema_beta_range
									lowerCamelCase__		:   List[str]  				=					ema_gamma_omega_range
									lowerCamelCase__		:   Union[str, Any]  				=					relative_positional_bias
									lowerCamelCase__		:   Any  				=					use_cache
									lowerCamelCase__		:   Dict  				=					classifier_dropout
									lowerCamelCase__		:   Union[str, Any]  				=					add_lm_hidden_dense_layer
									lowerCamelCase__		:   Union[str, Any]  				=					1  # not used but required by Hugging Face





class        __SCREAMING_SNAKE_CASE     (   _lowerCAmelCase							):







				@property
				def   lowercase_		(    self    :Any  )      ->					Mapping[str, Mapping[int, str]]:




									"""simple docstring"""



									if self.task == "multiple-choice":
														lowerCamelCase__		:   Tuple  				=					{0: '''batch''', 1: '''choice''', 2: '''sequence'''}
									else:
														lowerCamelCase__		:   str  				=					{0: '''batch''', 1: '''sequence'''}
									return OrderedDict(
									    [
									        ('''input_ids''', dynamic_axis),
									        ('''attention_mask''', dynamic_axis),
									    ]  )







"""simple docstring"""




import os
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device


if is_torch_available():
					import torch

					from transformers import GPTaLMHeadModel

@require_torch
@require_sentencepiece
@require_tokenizers
class        __SCREAMING_SNAKE_CASE     (   unittest.TestCase							):







				@slow
				@unittest.skip('''Model is not available.'''  )
				def   lowercase_		(    self    :Union[str, Any]  )      ->					Optional[int]:




									"""simple docstring"""



									lowerCamelCase__		:   Any  				=					'''nvidia/megatron-gpt2-345m/'''
									if "MYDIR" in os.environ:
														lowerCamelCase__		:   str  				=					os.path.join(os.environ['''MYDIR''']      ,__UpperCAmelCase  )
									lowerCamelCase__		:   Dict  				=					GPTaLMHeadModel.from_pretrained(__UpperCAmelCase  )
									model.to(__UpperCAmelCase  )
									model.half()

									lowerCamelCase__		:   Dict  				=					torch.tensor(
									    [[1_01, 71_10, 10_05, 10_56, 20_23, 1_13_33, 1_74_13, 10_29, 1_02]]      ,device=__UpperCAmelCase      ,dtype=torch.long      ,)

									with torch.no_grad():
														lowerCamelCase__		:   Tuple  				=					model(__UpperCAmelCase  ).logits

									lowerCamelCase__		:   List[str]  				=					torch.Size((1, 9, 5_02_57)  )
									self.assertEqual(output.shape      ,__UpperCAmelCase  )

									lowerCamelCase__		:   Tuple  				=					torch.tensor(
									    [
									        4.9_414,
									        -0.2_920,
									        -1.2_148,
									        -4.0_273,
									        -0.5_161,
									        -5.2_109,
									        -1.2_412,
									        -1.8_301,
									        -1.7_734,
									        -4.7_148,
									        -0.2_317,
									        -1.0_811,
									        -2.1_777,
									        0.4_141,
									        -3.7_969,
									        -4.0_586,
									        -2.5_332,
									        -3.3_809,
									        4.3_867,
									    ]      ,device=__UpperCAmelCase      ,dtype=torch.half      ,)

									for i in range(19  ):
														lowerCamelCase__				,						lowerCamelCase__		:   Dict  				=					8 * i // 17, 27_92 * i  # along the diagonal
														lowerCamelCase__				,						lowerCamelCase__		:   Optional[int]  				=					output[0, r, c], expected_diag[i]
														lowerCamelCase__		:   Dict  				=					F"""row={r} col={c} computed={computed} expected={expected}"""
														self.assertAlmostEqual(__UpperCAmelCase      ,__UpperCAmelCase      ,delta=1E-4      ,msg=__UpperCAmelCase  )






