
import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      MODEL_FOR_PRETRAINING_MAPPING,
      AlbertForMaskedLM,
      AlbertForMultipleChoice,
      AlbertForPreTraining,
      AlbertForQuestionAnswering,
      AlbertForSequenceClassification,
      AlbertForTokenClassification,
      AlbertModel,
  )
  from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST

class __snake_case		:
    def __init__(  self		,snake_case		,snake_case=13		,snake_case=7		,snake_case=True		,snake_case=True		,snake_case=True		,snake_case=True		,snake_case=99		,snake_case=16		,snake_case=36		,snake_case=6		,snake_case=6		,snake_case=6		,snake_case=37		,snake_case="gelu"		,snake_case=0.1		,snake_case=0.1		,snake_case=512		,snake_case=16		,snake_case=2		,snake_case=0.02		,snake_case=3		,snake_case=4		,snake_case=None		,):



           '''simple docstring'''







           lowercase							:     Union[str, Any]       	=      parent
           lowercase							:     Dict       	=      batch_size
           lowercase							:     Optional[int]       	=      seq_length
           lowercase							:     Union[str, Any]       	=      is_training
           lowercase							:     Dict       	=      use_input_mask
           lowercase							:     Dict       	=      use_token_type_ids
           lowercase							:     str       	=      use_labels
           lowercase							:     Union[str, Any]       	=      vocab_size
           lowercase							:     int       	=      embedding_size
           lowercase							:     List[str]       	=      hidden_size
           lowercase							:     Dict       	=      num_hidden_layers
           lowercase							:     Optional[Any]       	=      num_hidden_groups
           lowercase							:     List[Any]       	=      num_attention_heads
           lowercase							:     Union[str, Any]       	=      intermediate_size
           lowercase							:     Any       	=      hidden_act
           lowercase							:     Tuple       	=      hidden_dropout_prob
           lowercase							:     Dict       	=      attention_probs_dropout_prob
           lowercase							:     Any       	=      max_position_embeddings
           lowercase							:     List[str]       	=      type_vocab_size
           lowercase							:     int       	=      type_sequence_label_size
           lowercase							:     Optional[Any]       	=      initializer_range
           lowercase							:     int       	=      num_labels
           lowercase							:     Optional[Any]       	=      num_choices
           lowercase							:     Union[str, Any]       	=      scope
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     str       	=      ids_tensor([self.batch_size, self.seq_length]		,self.vocab_size      )

           lowercase							:     Union[str, Any]       	=      None
           if self.use_input_mask:
                  lowercase							:     Optional[Any]       	=      random_attention_mask([self.batch_size, self.seq_length]      )

           lowercase							:     Optional[int]       	=      None
           if self.use_token_type_ids:
                  lowercase							:     Any       	=      ids_tensor([self.batch_size, self.seq_length]		,self.type_vocab_size      )

           lowercase							:     Tuple       	=      None
           lowercase							:     Any       	=      None
           lowercase							:     Any       	=      None
           if self.use_labels:
                  lowercase							:     List[Any]       	=      ids_tensor([self.batch_size]		,self.type_sequence_label_size      )
                  lowercase							:     Dict       	=      ids_tensor([self.batch_size, self.seq_length]		,self.num_labels      )
                  lowercase							:     List[str]       	=      ids_tensor([self.batch_size]		,self.num_choices      )

           lowercase							:     Tuple       	=      self.get_config()

           return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           return AlbertConfig(
               vocab_size=self.vocab_size		,hidden_size=self.hidden_size		,num_hidden_layers=self.num_hidden_layers		,num_attention_heads=self.num_attention_heads		,intermediate_size=self.intermediate_size		,hidden_act=self.hidden_act		,hidden_dropout_prob=self.hidden_dropout_prob		,attention_probs_dropout_prob=self.attention_probs_dropout_prob		,max_position_embeddings=self.max_position_embeddings		,type_vocab_size=self.type_vocab_size		,initializer_range=self.initializer_range		,num_hidden_groups=self.num_hidden_groups		,)
    def       _SCREAMING_SNAKE_CASE       (  self		,snake_case		,snake_case		,snake_case		,snake_case		,snake_case		,snake_case		,snake_case      ):



           '''simple docstring'''







           lowercase							:     int       	=      AlbertModel(config=snake_case      )
           model.to(snake_case      )
           model.eval()
           lowercase							:     List[str]       	=      model(snake_case		,attention_mask=snake_case		,token_type_ids=snake_case      )
           lowercase							:     str       	=      model(snake_case		,token_type_ids=snake_case      )
           lowercase							:     Optional[Any]       	=      model(snake_case      )
           self.parent.assertEqual(result.last_hidden_state.shape		,(self.batch_size, self.seq_length, self.hidden_size)      )
           self.parent.assertEqual(result.pooler_output.shape		,(self.batch_size, self.hidden_size)      )
    def       _SCREAMING_SNAKE_CASE       (  self		,snake_case		,snake_case		,snake_case		,snake_case		,snake_case		,snake_case		,snake_case      ):



           '''simple docstring'''







           lowercase							:     Tuple       	=      AlbertForPreTraining(config=snake_case      )
           model.to(snake_case      )
           model.eval()
           lowercase							:     Union[str, Any]       	=      model(
               snake_case		,attention_mask=snake_case		,token_type_ids=snake_case		,labels=snake_case		,sentence_order_label=snake_case		,)
           self.parent.assertEqual(result.prediction_logits.shape		,(self.batch_size, self.seq_length, self.vocab_size)      )
           self.parent.assertEqual(result.sop_logits.shape		,(self.batch_size, config.num_labels)      )
    def       _SCREAMING_SNAKE_CASE       (  self		,snake_case		,snake_case		,snake_case		,snake_case		,snake_case		,snake_case		,snake_case      ):



           '''simple docstring'''







           lowercase							:     int       	=      AlbertForMaskedLM(config=snake_case      )
           model.to(snake_case      )
           model.eval()
           lowercase							:     int       	=      model(snake_case		,attention_mask=snake_case		,token_type_ids=snake_case		,labels=snake_case      )
           self.parent.assertEqual(result.logits.shape		,(self.batch_size, self.seq_length, self.vocab_size)      )
    def       _SCREAMING_SNAKE_CASE       (  self		,snake_case		,snake_case		,snake_case		,snake_case		,snake_case		,snake_case		,snake_case      ):



           '''simple docstring'''







           lowercase							:     str       	=      AlbertForQuestionAnswering(config=snake_case      )
           model.to(snake_case      )
           model.eval()
           lowercase							:     List[str]       	=      model(
               snake_case		,attention_mask=snake_case		,token_type_ids=snake_case		,start_positions=snake_case		,end_positions=snake_case		,)
           self.parent.assertEqual(result.start_logits.shape		,(self.batch_size, self.seq_length)      )
           self.parent.assertEqual(result.end_logits.shape		,(self.batch_size, self.seq_length)      )
    def       _SCREAMING_SNAKE_CASE       (  self		,snake_case		,snake_case		,snake_case		,snake_case		,snake_case		,snake_case		,snake_case      ):



           '''simple docstring'''







           lowercase							:     str       	=      self.num_labels
           lowercase							:     Tuple       	=      AlbertForSequenceClassification(snake_case      )
           model.to(snake_case      )
           model.eval()
           lowercase							:     Tuple       	=      model(snake_case		,attention_mask=snake_case		,token_type_ids=snake_case		,labels=snake_case      )
           self.parent.assertEqual(result.logits.shape		,(self.batch_size, self.num_labels)      )
    def       _SCREAMING_SNAKE_CASE       (  self		,snake_case		,snake_case		,snake_case		,snake_case		,snake_case		,snake_case		,snake_case      ):



           '''simple docstring'''







           lowercase							:     Tuple       	=      self.num_labels
           lowercase							:     Optional[int]       	=      AlbertForTokenClassification(config=snake_case      )
           model.to(snake_case      )
           model.eval()
           lowercase							:     int       	=      model(snake_case		,attention_mask=snake_case		,token_type_ids=snake_case		,labels=snake_case      )
           self.parent.assertEqual(result.logits.shape		,(self.batch_size, self.seq_length, self.num_labels)      )
    def       _SCREAMING_SNAKE_CASE       (  self		,snake_case		,snake_case		,snake_case		,snake_case		,snake_case		,snake_case		,snake_case      ):



           '''simple docstring'''







           lowercase							:     Any       	=      self.num_choices
           lowercase							:     Any       	=      AlbertForMultipleChoice(config=snake_case      )
           model.to(snake_case      )
           model.eval()
           lowercase							:     List[Any]       	=      input_ids.unsqueeze(1      ).expand(-1		,self.num_choices		,-1      ).contiguous()
           lowercase							:     Tuple       	=      token_type_ids.unsqueeze(1      ).expand(-1		,self.num_choices		,-1      ).contiguous()
           lowercase							:     Tuple       	=      input_mask.unsqueeze(1      ).expand(-1		,self.num_choices		,-1      ).contiguous()
           lowercase							:     Dict       	=      model(
               snake_case		,attention_mask=snake_case		,token_type_ids=snake_case		,labels=snake_case		,)
           self.parent.assertEqual(result.logits.shape		,(self.batch_size, self.num_choices)      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Tuple       	=      self.prepare_config_and_inputs()
           (
               (
               lowercase
           )	,			(
               lowercase
           )	,			(
               lowercase
           )	,			(
               lowercase
           )	,			(
               lowercase
           )	,			(
               lowercase
           )	,			(
               lowercase
           )	,			
           )							:     str       	=      config_and_inputs
           lowercase							:     int       	=      {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
           return config, inputs_dict






@require_torch
class __snake_case		(      lowerCAmelCase					,	lowerCAmelCase					,	unittest.TestCase     ):
    _a		:						Tuple=				(
        (
            AlbertModel,
            AlbertForPreTraining,
            AlbertForMaskedLM,
            AlbertForMultipleChoice,
            AlbertForSequenceClassification,
            AlbertForTokenClassification,
            AlbertForQuestionAnswering,
        )
        if is_torch_available()
        else ()
    )
    _a		:						List[str]=				(
        {
            "feature-extraction": AlbertModel,
            "fill-mask": AlbertForMaskedLM,
            "question-answering": AlbertForQuestionAnswering,
            "text-classification": AlbertForSequenceClassification,
            "token-classification": AlbertForTokenClassification,
            "zero-shot": AlbertForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )
    _a		:						Dict=				True
    def       _SCREAMING_SNAKE_CASE       (  self		,snake_case		,snake_case		,snake_case=False      ):



           '''simple docstring'''







           lowercase							:     List[str]       	=      super()._prepare_for_class(snake_case		,snake_case		,return_labels=snake_case      )

           if return_labels:
                  if model_class in get_values(snake_case      ):
                         lowercase							:     List[Any]       	=      torch.zeros(
                             (self.model_tester.batch_size, self.model_tester.seq_length)		,dtype=torch.long		,device=snake_case      )
                         lowercase							:     str       	=      torch.zeros(
                             self.model_tester.batch_size		,dtype=torch.long		,device=snake_case      )
           return inputs_dict
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     str       	=      AlbertModelTester(self      )
           lowercase							:     Optional[int]       	=      ConfigTester(self		,config_class=snake_case		,hidden_size=37      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           self.config_tester.run_common_tests()
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Dict       	=      self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*snake_case      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Union[str, Any]       	=      self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_pretraining(*snake_case      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Dict       	=      self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_masked_lm(*snake_case      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     List[Any]       	=      self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_multiple_choice(*snake_case      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Optional[int]       	=      self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_question_answering(*snake_case      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Tuple       	=      self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_sequence_classification(*snake_case      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     str       	=      self.model_tester.prepare_config_and_inputs()
           for type in ["absolute", "relative_key", "relative_key_query"]:
                  lowercase							:     Optional[Any]       	=      type
                  self.model_tester.create_and_check_model(*snake_case      )
    @slow
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                  lowercase							:     List[Any]       	=      AlbertModel.from_pretrained(snake_case      )
                  self.assertIsNotNone(snake_case      )






@require_torch
class __snake_case		(      unittest.TestCase     ):
    @slow
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Optional[int]       	=      AlbertModel.from_pretrained("""albert-base-v2"""      )
           lowercase							:     int       	=      torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]      )
           lowercase							:     Optional[int]       	=      torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]      )
           with torch.no_grad():
                  lowercase							:     List[str]       	=      model(snake_case		,attention_mask=snake_case      )[0]
           lowercase							:     Any       	=      torch.Size((1, 11, 768)      )
           self.assertEqual(output.shape		,snake_case      )
           lowercase							:     int       	=      torch.tensor(
               [[[-0.6_513, 1.5_035, -0.2_766], [-0.6_515, 1.5_046, -0.2_780], [-0.6_512, 1.5_049, -0.2_784]]]      )

           self.assertTrue(torch.allclose(output[:, 1:4, 1:4]		,snake_case		,atol=1e-4      )      )


from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast
from ...utils import logging


lowercase  :					List[str]						    =							logging.get_logger(__name__)

lowercase  :					Optional[Any]						    =							{
    """EleutherAI/gpt-neo-1.3B""": """https://huggingface.co/EleutherAI/gpt-neo-1.3B/resolve/main/config.json""",
    # See all GPTNeo models at https://huggingface.co/models?filter=gpt_neo
}

class __snake_case		(      lowerCAmelCase     ):
    _a		:						str=				"gpt_neo"
    _a		:						Optional[int]=				["past_key_values"]
    _a		:						Dict=				{"num_attention_heads": "num_heads", "num_hidden_layers": "num_layers"}
    def __init__(  self		,snake_case=50257		,snake_case=2048		,snake_case=2048		,snake_case=24		,snake_case=[[["global", "local"], 12]]		,snake_case=16		,snake_case=None		,snake_case=256		,snake_case="gelu_new"		,snake_case=0.0		,snake_case=0.0		,snake_case=0.0		,snake_case=0.1		,snake_case=1e-5		,snake_case=0.02		,snake_case=True		,snake_case=50256		,snake_case=50256		,**snake_case		,):



           '''simple docstring'''







           lowercase							:     int       	=      vocab_size
           lowercase							:     Union[str, Any]       	=      max_position_embeddings
           lowercase							:     Dict       	=      hidden_size
           lowercase							:     Union[str, Any]       	=      num_layers
           lowercase							:     Union[str, Any]       	=      num_heads
           lowercase							:     Optional[int]       	=      intermediate_size
           lowercase							:     List[str]       	=      window_size
           lowercase							:     Optional[int]       	=      activation_function
           lowercase							:     List[str]       	=      resid_dropout
           lowercase							:     int       	=      embed_dropout
           lowercase							:     Optional[int]       	=      attention_dropout
           lowercase							:     Tuple       	=      classifier_dropout
           lowercase							:     Optional[int]       	=      layer_norm_epsilon
           lowercase							:     Dict       	=      initializer_range
           lowercase							:     List[str]       	=      use_cache

           lowercase							:     Optional[int]       	=      bos_token_id
           lowercase							:     int       	=      eos_token_id

           lowercase							:     Union[str, Any]       	=      attention_types
           lowercase							:     Dict       	=      self.expand_attention_types_params(snake_case      )

           if len(self.attention_layers      ) != self.num_layers:
                  raise ValueError(
                      """Configuration for convolutional module is incorrect. """
                      """It is required that `len(config.attention_layers)` == `config.num_layers` """
                      f"but is `len(config.attention_layers) = {len(self.attention_layers      )}`, "
                      f"`config.num_layers = {self.num_layers}`. "
                      """`config.attention_layers` is prepared using `config.attention_types`. """
                      """Please verify the value of `config.attention_types` argument."""      )

           super().__init__(bos_token_id=snake_case		,eos_token_id=snake_case		,**snake_case      )
    @staticmethod
    def       _SCREAMING_SNAKE_CASE       (  snake_case      ):



           '''simple docstring'''







           lowercase							:     List[Any]       	=      []
           for item in attention_types:
                  for _ in range(item[1]      ):
                         attentions.extend(item[0]      )
           return attentions







def 						_snake_case(  SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__				) ->    Tuple:
       import torch

       lowercase							:     Tuple       	=      input.size()
       lowercase							:     int       	=      len(SCREAMING_SNAKE_CASE__				)
       lowercase							:     Union[str, Any]       	=      shape[dimension]

       lowercase							:     int       	=      torch.arange(0					,	SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__				)
       lowercase							:     Optional[Any]       	=      torch.div(sizedim - size					,	SCREAMING_SNAKE_CASE__					,	rounding_mode="""floor"""				) + 1
       lowercase							:     Dict       	=      torch.arange(SCREAMING_SNAKE_CASE__				) + low_indices[:min_length][:, None]

       lowercase							:     Union[str, Any]       	=      [slice(SCREAMING_SNAKE_CASE__				)] * rank
       lowercase							:     Optional[Any]       	=      indices
       lowercase							:     List[str]       	=      input[s]

       lowercase							:     Optional[int]       	=      list(range(0					,	rank + 1				)				)
       perm.append(perm.pop(dimension + 1				)				)

       return sliced.permute(SCREAMING_SNAKE_CASE__				)







def 						_snake_case(  SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__				) ->    Union[str, Any]:
       import torch

       lowercase							:     Union[str, Any]       	=      torch.arange(1					,	SCREAMING_SNAKE_CASE__				)
       lowercase							:     Optional[Any]       	=      torch.remainder(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__				)
       lowercase							:     List[Any]       	=      remainders == 0
       lowercase							:     Optional[int]       	=      candidates[divisor_indices]
       lowercase							:     List[Any]       	=      torch.max(SCREAMING_SNAKE_CASE__				)
       return largest_divisor, torch.div(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	rounding_mode="""floor"""				)




class __snake_case		(      lowerCAmelCase     ):
    @property
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Dict       	=      OrderedDict({"""input_ids""": {0: """batch""", 1: """sequence"""}}      )
           if self.use_past:
                  self.fill_with_past_key_values_(snake_case		,direction="""inputs"""      )
                  lowercase							:     Dict       	=      {0: """batch""", 1: """past_sequence + sequence"""}
           else:
                  lowercase							:     List[str]       	=      {0: """batch""", 1: """sequence"""}

           return common_inputs
    @property
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           return self._config.num_heads
    def       _SCREAMING_SNAKE_CASE       (  self		,snake_case		,snake_case = -1		,snake_case = -1		,snake_case = False		,snake_case = None		,):



           '''simple docstring'''







           lowercase							:     Any       	=      super(snake_case		,self      ).generate_dummy_inputs(
               snake_case		,batch_size=snake_case		,seq_length=snake_case		,is_pair=snake_case		,framework=snake_case      )

           # We need to order the input in the way they appears in the forward()
           lowercase							:     List[str]       	=      OrderedDict({"""input_ids""": common_inputs["""input_ids"""]}      )

           # Need to add the past_keys
           if self.use_past:
                  if not is_torch_available():
                         raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""      )
                  else:
                         import torch

                         lowercase	,			lowercase							:     List[Any]       	=      common_inputs["""input_ids"""].shape
                         # Not using the same length for past_key_values
                         lowercase							:     Optional[int]       	=      seqlen + 2
                         lowercase							:     int       	=      (
                             batch,
                             self.num_attention_heads,
                             past_key_values_length,
                             self._config.hidden_size // self.num_attention_heads,
                         )
                         lowercase							:     Optional[Any]       	=      [
                             (torch.zeros(snake_case      ), torch.zeros(snake_case      )) for _ in range(self.num_layers      )
                         ]

           lowercase							:     Optional[Any]       	=      common_inputs["""attention_mask"""]
           if self.use_past:
                  lowercase							:     Any       	=      ordered_inputs["""attention_mask"""].dtype
                  lowercase							:     Union[str, Any]       	=      torch.cat(
                      [ordered_inputs["""attention_mask"""], torch.ones(snake_case		,snake_case		,dtype=snake_case      )]		,dim=1      )

           return ordered_inputs
    @property
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           return 13
