

import os
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device


if is_torch_available():
      import torch

      from transformers import GPTaLMHeadModel


@require_torch
@require_sentencepiece
@require_tokenizers
class      lowerCamelCase_ (			unittest.TestCase):




 @slow
 @unittest.skip('Model is not available.'							)
 def     a       (			self							):
   __UpperCAmelCase	:  Dict		=     'nvidia/megatron-gpt2-345m/'
   if "MYDIR" in os.environ:
     __UpperCAmelCase	:  List[Any]		=     os.path.join(os.environ['MYDIR']  ,  _UpperCAmelCase							)
   __UpperCAmelCase	:  Optional[int]		=     GPTaLMHeadModel.from_pretrained(_UpperCAmelCase							)
   model.to(_UpperCAmelCase							)
   model.half()

   __UpperCAmelCase	:  Optional[int]		=     torch.tensor(
       [[101, 7_110, 1_005, 1_056, 2_023, 11_333, 17_413, 1_029, 102]]  ,  device=_UpperCAmelCase  ,  dtype=torch.long  ,  )

   with torch.no_grad():
     __UpperCAmelCase	:  Any		=     model(_UpperCAmelCase							).logits

   __UpperCAmelCase	:  Tuple		=     torch.Size((1, 9, 50_257)							)
   self.assertEqual(output.shape  ,  _UpperCAmelCase							)

   __UpperCAmelCase	:  Any		=     torch.tensor(
       [
           4.9_4_1_4,
           -0.2_9_2_0,
           -1.2_1_4_8,
           -4.0_2_7_3,
           -0.5_1_6_1,
           -5.2_1_0_9,
           -1.2_4_1_2,
           -1.8_3_0_1,
           -1.7_7_3_4,
           -4.7_1_4_8,
           -0.2_3_1_7,
           -1.0_8_1_1,
           -2.1_7_7_7,
           0.4_1_4_1,
           -3.7_9_6_9,
           -4.0_5_8_6,
           -2.5_3_3_2,
           -3.3_8_0_9,
           4.3_8_6_7,
       ]  ,  device=_UpperCAmelCase  ,  dtype=torch.half  ,  )

   for i in range(19							):
     __UpperCAmelCase,			__UpperCAmelCase	:  Optional[int]		=     8 * i // 17, 2_792 * i  # along the diagonal
     __UpperCAmelCase,			__UpperCAmelCase	:  Optional[Any]		=     output[0, r, c], expected_diag[i]
     __UpperCAmelCase	:  Optional[Any]		=     F"row={r} col={c} computed={computed} expected={expected}"
     self.assertAlmostEqual(_UpperCAmelCase  ,  _UpperCAmelCase  ,  delta=1E-4  ,  msg=_UpperCAmelCase							)









from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCamelCase			:					List[Any]         =							logging.get_logger(__name__)

UpperCamelCase			:					Tuple         =							{
    '''mnaylor/mega-base-wikitext''': '''https://huggingface.co/mnaylor/mega-base-wikitext/resolve/main/config.json''',
}

class      __snake_case				(   _UpperCAmelCase):
					SCREAMING_SNAKE_CASE_		:       str								=			"mega"
					def __init__(						self       :						Optional[int]	,      _lowerCAmelCase       :						List[Any]=3_0_5_2_2	,      _lowerCAmelCase       :						List[str]=1_2_8	,      _lowerCAmelCase       :						Dict=4	,      _lowerCAmelCase       :						Optional[Any]=2_5_6	,      _lowerCAmelCase       :						Optional[Any]=1_6	,      _lowerCAmelCase       :						Dict=True	,      _lowerCAmelCase       :						Optional[Any]=6_4	,      _lowerCAmelCase       :						Optional[Any]=False	,      _lowerCAmelCase       :						Optional[int]=-1	,      _lowerCAmelCase       :						Any=None	,      _lowerCAmelCase       :						List[Any]=True	,      _lowerCAmelCase       :						List[Any]="scalenorm"	,      _lowerCAmelCase       :						str=True	,      _lowerCAmelCase       :						Union[str, Any]="silu"	,      _lowerCAmelCase       :						Optional[Any]="softmax"	,      _lowerCAmelCase       :						Optional[Any]=0.1	,      _lowerCAmelCase       :						Dict=0.1	,      _lowerCAmelCase       :						Any=0.1	,      _lowerCAmelCase       :						Optional[Any]=False	,      _lowerCAmelCase       :						Any=True	,      _lowerCAmelCase       :						int=2_5_6	,      _lowerCAmelCase       :						List[str]=True	,      _lowerCAmelCase       :						str=0.1	,      _lowerCAmelCase       :						List[str]=2_0_4_8	,      _lowerCAmelCase       :						Optional[Any]=False	,      _lowerCAmelCase       :						str=2	,      _lowerCAmelCase       :						Dict=0.02	,      _lowerCAmelCase       :						str=0.2	,      _lowerCAmelCase       :						List[str]=0.02	,      _lowerCAmelCase       :						int=1.0	,      _lowerCAmelCase       :						Optional[int]=1	,      _lowerCAmelCase       :						List[str]=0	,      _lowerCAmelCase       :						str=2	,      _lowerCAmelCase       :						Optional[Any]="rotary"	,      _lowerCAmelCase       :						int=None	,      _lowerCAmelCase       :						int=True	,      _lowerCAmelCase       :						Optional[Any]=True	,      **_lowerCAmelCase       :						str	,      )							->					Any:



											'''simple docstring'''





											super().__init__(pad_token_id=_lowerCAmelCase	,      bos_token_id=_lowerCAmelCase	,      eos_token_id=_lowerCAmelCase	,      **_lowerCAmelCase  )

											__magic_name__					=      vocab_size
											__magic_name__					=      hidden_size
											__magic_name__					=      num_hidden_layers
											__magic_name__					=      activation
											__magic_name__					=      attention_activation
											__magic_name__					=      intermediate_size
											__magic_name__					=      ema_projection_size
											__magic_name__					=      bidirectional
											__magic_name__					=      shared_representation_size
											__magic_name__					=      use_chunking
											__magic_name__					=      chunk_size
											__magic_name__					=      truncation
											__magic_name__					=      normalize_before_mega
											__magic_name__					=      normalization_type
											__magic_name__					=      norm_affine
											__magic_name__					=      dropout_prob
											__magic_name__					=      hidden_dropout_prob
											__magic_name__					=      attention_probs_dropout_prob
											__magic_name__					=      use_feature_dropout
											__magic_name__					=      use_normalized_ffn
											__magic_name__					=      nffn_hidden_size
											__magic_name__					=      normalize_before_ffn
											__magic_name__					=      nffn_activation_dropout_prob
											__magic_name__					=      max_positions
											__magic_name__					=      add_token_type_embeddings
											__magic_name__					=      type_vocab_size
											__magic_name__					=      initializer_range
											__magic_name__					=      ema_delta_alpha_range
											__magic_name__					=      ema_beta_range
											__magic_name__					=      ema_gamma_omega_range
											__magic_name__					=      relative_positional_bias
											__magic_name__					=      use_cache
											__magic_name__					=      classifier_dropout
											__magic_name__					=      add_lm_hidden_dense_layer
											__magic_name__					=      1  # not used but required by Hugging Face







class      __snake_case				(   _UpperCAmelCase):
					@property
					def 	a			(						self       :						int  )							->					Mapping[str, Mapping[int, str]]:



											'''simple docstring'''





											if self.task == "multiple-choice":
																	__magic_name__					=      {0: 'batch', 1: 'choice', 2: 'sequence'}
											else:
																	__magic_name__					=      {0: 'batch', 1: 'sequence'}
											return OrderedDict(
											    [
											        ('input_ids', dynamic_axis),
											        ('attention_mask', dynamic_axis),
											    ]  )

