
'''simple docstring'''







import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import (
					    MODEL_FOR_PRETRAINING_MAPPING,
					    AlbertForMaskedLM,
					    AlbertForMultipleChoice,
					    AlbertForPreTraining,
					    AlbertForQuestionAnswering,
					    AlbertForSequenceClassification,
					    AlbertForTokenClassification,
					    AlbertModel,
					)
					from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST


class 						__UpperCAmelCase      :


							def __init__(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_=13    ,   lowerCAmelCase_=7    ,   lowerCAmelCase_=True    ,   lowerCAmelCase_=True    ,   lowerCAmelCase_=True    ,   lowerCAmelCase_=True    ,   lowerCAmelCase_=99    ,   lowerCAmelCase_=16    ,   lowerCAmelCase_=36    ,   lowerCAmelCase_=6    ,   lowerCAmelCase_=6    ,   lowerCAmelCase_=6    ,   lowerCAmelCase_=37    ,   lowerCAmelCase_="gelu"    ,   lowerCAmelCase_=0.1    ,   lowerCAmelCase_=0.1    ,   lowerCAmelCase_=5_12    ,   lowerCAmelCase_=16    ,   lowerCAmelCase_=2    ,   lowerCAmelCase_=0.02    ,   lowerCAmelCase_=3    ,   lowerCAmelCase_=4    ,   lowerCAmelCase_=None    ,   ):





								"""simple docstring"""






								_snake_case   						=		parent
								_snake_case   						=		batch_size
								_snake_case   						=		seq_length
								_snake_case   						=		is_training
								_snake_case   						=		use_input_mask
								_snake_case   						=		use_token_type_ids
								_snake_case   						=		use_labels
								_snake_case   						=		vocab_size
								_snake_case   						=		embedding_size
								_snake_case   						=		hidden_size
								_snake_case   						=		num_hidden_layers
								_snake_case   						=		num_hidden_groups
								_snake_case   						=		num_attention_heads
								_snake_case   						=		intermediate_size
								_snake_case   						=		hidden_act
								_snake_case   						=		hidden_dropout_prob
								_snake_case   						=		attention_probs_dropout_prob
								_snake_case   						=		max_position_embeddings
								_snake_case   						=		type_vocab_size
								_snake_case   						=		type_sequence_label_size
								_snake_case   						=		initializer_range
								_snake_case   						=		num_labels
								_snake_case   						=		num_choices
								_snake_case   						=		scope


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		ids_tensor([self.batch_size, self.seq_length]    ,   self.vocab_size  )

								_snake_case   						=		None
								if self.use_input_mask:
									_snake_case   						=		random_attention_mask([self.batch_size, self.seq_length]  )

								_snake_case   						=		None
								if self.use_token_type_ids:
									_snake_case   						=		ids_tensor([self.batch_size, self.seq_length]    ,   self.type_vocab_size  )

								_snake_case   						=		None
								_snake_case   						=		None
								_snake_case   						=		None
								if self.use_labels:
									_snake_case   						=		ids_tensor([self.batch_size]    ,   self.type_sequence_label_size  )
									_snake_case   						=		ids_tensor([self.batch_size, self.seq_length]    ,   self.num_labels  )
									_snake_case   						=		ids_tensor([self.batch_size]    ,   self.num_choices  )

								_snake_case   						=		self.get_config()

								return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								return AlbertConfig(
								    vocab_size=self.vocab_size    ,   hidden_size=self.hidden_size    ,   num_hidden_layers=self.num_hidden_layers    ,   num_attention_heads=self.num_attention_heads    ,   intermediate_size=self.intermediate_size    ,   hidden_act=self.hidden_act    ,   hidden_dropout_prob=self.hidden_dropout_prob    ,   attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,   max_position_embeddings=self.max_position_embeddings    ,   type_vocab_size=self.type_vocab_size    ,   initializer_range=self.initializer_range    ,   num_hidden_groups=self.num_hidden_groups    ,   )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		AlbertModel(config=lowerCAmelCase_  )
								model.to(lowerCAmelCase_  )
								model.eval()
								_snake_case   						=		model(lowerCAmelCase_    ,   attention_mask=lowerCAmelCase_    ,   token_type_ids=lowerCAmelCase_  )
								_snake_case   						=		model(lowerCAmelCase_    ,   token_type_ids=lowerCAmelCase_  )
								_snake_case   						=		model(lowerCAmelCase_  )
								self.parent.assertEqual(result.last_hidden_state.shape    ,   (self.batch_size, self.seq_length, self.hidden_size)  )
								self.parent.assertEqual(result.pooler_output.shape    ,   (self.batch_size, self.hidden_size)  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		AlbertForPreTraining(config=lowerCAmelCase_  )
								model.to(lowerCAmelCase_  )
								model.eval()
								_snake_case   						=		model(
								    lowerCAmelCase_    ,   attention_mask=lowerCAmelCase_    ,   token_type_ids=lowerCAmelCase_    ,   labels=lowerCAmelCase_    ,   sentence_order_label=lowerCAmelCase_    ,   )
								self.parent.assertEqual(result.prediction_logits.shape    ,   (self.batch_size, self.seq_length, self.vocab_size)  )
								self.parent.assertEqual(result.sop_logits.shape    ,   (self.batch_size, config.num_labels)  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		AlbertForMaskedLM(config=lowerCAmelCase_  )
								model.to(lowerCAmelCase_  )
								model.eval()
								_snake_case   						=		model(lowerCAmelCase_    ,   attention_mask=lowerCAmelCase_    ,   token_type_ids=lowerCAmelCase_    ,   labels=lowerCAmelCase_  )
								self.parent.assertEqual(result.logits.shape    ,   (self.batch_size, self.seq_length, self.vocab_size)  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		AlbertForQuestionAnswering(config=lowerCAmelCase_  )
								model.to(lowerCAmelCase_  )
								model.eval()
								_snake_case   						=		model(
								    lowerCAmelCase_    ,   attention_mask=lowerCAmelCase_    ,   token_type_ids=lowerCAmelCase_    ,   start_positions=lowerCAmelCase_    ,   end_positions=lowerCAmelCase_    ,   )
								self.parent.assertEqual(result.start_logits.shape    ,   (self.batch_size, self.seq_length)  )
								self.parent.assertEqual(result.end_logits.shape    ,   (self.batch_size, self.seq_length)  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		self.num_labels
								_snake_case   						=		AlbertForSequenceClassification(lowerCAmelCase_  )
								model.to(lowerCAmelCase_  )
								model.eval()
								_snake_case   						=		model(lowerCAmelCase_    ,   attention_mask=lowerCAmelCase_    ,   token_type_ids=lowerCAmelCase_    ,   labels=lowerCAmelCase_  )
								self.parent.assertEqual(result.logits.shape    ,   (self.batch_size, self.num_labels)  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		self.num_labels
								_snake_case   						=		AlbertForTokenClassification(config=lowerCAmelCase_  )
								model.to(lowerCAmelCase_  )
								model.eval()
								_snake_case   						=		model(lowerCAmelCase_    ,   attention_mask=lowerCAmelCase_    ,   token_type_ids=lowerCAmelCase_    ,   labels=lowerCAmelCase_  )
								self.parent.assertEqual(result.logits.shape    ,   (self.batch_size, self.seq_length, self.num_labels)  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		self.num_choices
								_snake_case   						=		AlbertForMultipleChoice(config=lowerCAmelCase_  )
								model.to(lowerCAmelCase_  )
								model.eval()
								_snake_case   						=		input_ids.unsqueeze(1  ).expand(-1    ,   self.num_choices    ,   -1  ).contiguous()
								_snake_case   						=		token_type_ids.unsqueeze(1  ).expand(-1    ,   self.num_choices    ,   -1  ).contiguous()
								_snake_case   						=		input_mask.unsqueeze(1  ).expand(-1    ,   self.num_choices    ,   -1  ).contiguous()
								_snake_case   						=		model(
								    lowerCAmelCase_    ,   attention_mask=lowerCAmelCase_    ,   token_type_ids=lowerCAmelCase_    ,   labels=lowerCAmelCase_    ,   )
								self.parent.assertEqual(result.logits.shape    ,   (self.batch_size, self.num_choices)  )




							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.prepare_config_and_inputs()
								(
								    (
								    _snake_case
								)						, (
								    _snake_case
								)						, (
								    _snake_case
								)						, (
								    _snake_case
								)						, (
								    _snake_case
								)						, (
								    _snake_case
								)						, (
								    _snake_case
								)						, 
								)   						=		config_and_inputs
								_snake_case   						=		{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
								return config, inputs_dict


@require_torch
class 						__UpperCAmelCase      (					_lowerCamelCase						,   _lowerCamelCase						,   unittest.TestCase						):
							__lowercase      		=       (
							    (
							        AlbertModel,
							        AlbertForPreTraining,
							        AlbertForMaskedLM,
							        AlbertForMultipleChoice,
							        AlbertForSequenceClassification,
							        AlbertForTokenClassification,
							        AlbertForQuestionAnswering,
							    )
							    if is_torch_available()
							    else ()
							)
							__lowercase      		=       (
							    {
							        """feature-extraction""": AlbertModel,
							        """fill-mask""": AlbertForMaskedLM,
							        """question-answering""": AlbertForQuestionAnswering,
							        """text-classification""": AlbertForSequenceClassification,
							        """token-classification""": AlbertForTokenClassification,
							        """zero-shot""": AlbertForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							__lowercase      		=       True


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_=False  ):





								"""simple docstring"""






								_snake_case   						=		super()._prepare_for_class(lowerCAmelCase_    ,   lowerCAmelCase_    ,   return_labels=lowerCAmelCase_  )

								if return_labels:
									if model_class in get_values(lowerCAmelCase_  ):
										_snake_case   						=		torch.zeros(
										    (self.model_tester.batch_size, self.model_tester.seq_length)    ,   dtype=torch.long    ,   device=lowerCAmelCase_  )
										_snake_case   						=		torch.zeros(
										    self.model_tester.batch_size    ,   dtype=torch.long    ,   device=lowerCAmelCase_  )
								return inputs_dict


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		AlbertModelTester(self  )
								_snake_case   						=		ConfigTester(self    ,   config_class=lowerCAmelCase_    ,   hidden_size=37  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								self.config_tester.run_common_tests()


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*lowerCAmelCase_  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_pretraining(*lowerCAmelCase_  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase_  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_multiple_choice(*lowerCAmelCase_  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase_  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase_  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.model_tester.prepare_config_and_inputs()
								for type in ["absolute", "relative_key", "relative_key_query"]:
									_snake_case   						=		type
									self.model_tester.create_and_check_model(*lowerCAmelCase_  )




							@slow
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									_snake_case   						=		AlbertModel.from_pretrained(lowerCAmelCase_  )
									self.assertIsNotNone(lowerCAmelCase_  )


@require_torch
class 						__UpperCAmelCase      (					unittest.TestCase						):


							@slow
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		AlbertModel.from_pretrained('albert-base-v2'  )
								_snake_case   						=		torch.tensor([[0, 3_45, 2_32, 3_28, 7_40, 1_40, 16_95, 69, 60_78, 15_88, 2]]  )
								_snake_case   						=		torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]  )
								with torch.no_grad():
									_snake_case   						=		model(lowerCAmelCase_    ,   attention_mask=lowerCAmelCase_  )[0]
								_snake_case   						=		torch.Size((1, 11, 7_68)  )
								self.assertEqual(output.shape    ,   lowerCAmelCase_  )
								_snake_case   						=		torch.tensor(
								    [[[-0.6513, 1.5035, -0.2766], [-0.6515, 1.5046, -0.2780], [-0.6512, 1.5049, -0.2784]]]  )

								self.assertTrue(torch.allclose(output[:, 1:4, 1:4]    ,   lowerCAmelCase_    ,   atol=1E-4  )  )



'''simple docstring'''







import warnings

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase    :							int      =					logging.get_logger(__name__)

lowercase    :							Union[str, Any]      =					{
    "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/config.json",
    "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/config.json",
}


class 						__UpperCAmelCase      (					_lowerCamelCase						):
							__lowercase      		=       """xlnet"""
							__lowercase      		=       ["""mems"""]
							__lowercase      		=       {
							    """n_token""": """vocab_size""",  # Backward compatibility
							    """hidden_size""": """d_model""",
							    """num_attention_heads""": """n_head""",
							    """num_hidden_layers""": """n_layer""",
							}


							def __init__(       self    ,   lowerCAmelCase_=3_20_00    ,   lowerCAmelCase_=10_24    ,   lowerCAmelCase_=24    ,   lowerCAmelCase_=16    ,   lowerCAmelCase_=40_96    ,   lowerCAmelCase_="gelu"    ,   lowerCAmelCase_=True    ,   lowerCAmelCase_="bi"    ,   lowerCAmelCase_=0.02    ,   lowerCAmelCase_=1E-12    ,   lowerCAmelCase_=0.1    ,   lowerCAmelCase_=5_12    ,   lowerCAmelCase_=None    ,   lowerCAmelCase_=True    ,   lowerCAmelCase_=False    ,   lowerCAmelCase_=False    ,   lowerCAmelCase_=-1    ,   lowerCAmelCase_=False    ,   lowerCAmelCase_="last"    ,   lowerCAmelCase_=True    ,   lowerCAmelCase_="tanh"    ,   lowerCAmelCase_=0.1    ,   lowerCAmelCase_=5    ,   lowerCAmelCase_=5    ,   lowerCAmelCase_=5    ,   lowerCAmelCase_=1    ,   lowerCAmelCase_=2    ,   **lowerCAmelCase_    ,   ):





								"""simple docstring"""






								_snake_case   						=		vocab_size
								_snake_case   						=		d_model
								_snake_case   						=		n_layer
								_snake_case   						=		n_head
								if d_model % n_head != 0:
									raise ValueError(F'\'d_model % n_head\' ({d_model % n_head}) should be equal to 0'  )
								if "d_head" in kwargs:
									if kwargs["d_head"] != d_model // n_head:
										raise ValueError(
										    F'`d_head` ({kwargs["d_head"]}) should be equal to `d_model // n_head` ({d_model // n_head})'  )
								_snake_case   						=		d_model // n_head
								_snake_case   						=		ff_activation
								_snake_case   						=		d_inner
								_snake_case   						=		untie_r
								_snake_case   						=		attn_type

								_snake_case   						=		initializer_range
								_snake_case   						=		layer_norm_eps

								_snake_case   						=		dropout
								_snake_case   						=		mem_len
								_snake_case   						=		reuse_len
								_snake_case   						=		bi_data
								_snake_case   						=		clamp_len
								_snake_case   						=		same_length

								_snake_case   						=		summary_type
								_snake_case   						=		summary_use_proj
								_snake_case   						=		summary_activation
								_snake_case   						=		summary_last_dropout
								_snake_case   						=		start_n_top
								_snake_case   						=		end_n_top

								_snake_case   						=		bos_token_id
								_snake_case   						=		pad_token_id
								_snake_case   						=		eos_token_id

								if "use_cache" in kwargs:
									warnings.warn(
									    'The `use_cache` argument is deprecated and will be removed in a future version, use `use_mems_eval`'
									    ' instead.'    ,   lowerCAmelCase_    ,   )
									_snake_case   						=		kwargs['use_cache']

								_snake_case   						=		use_mems_eval
								_snake_case   						=		use_mems_train
								super().__init__(pad_token_id=lowerCAmelCase_    ,   bos_token_id=lowerCAmelCase_    ,   eos_token_id=lowerCAmelCase_    ,   **lowerCAmelCase_  )


							@property
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								logger.info(F'The model {self.model_type} is one of the few models that has no sequence length limit.'  )
								return -1




							@max_position_embeddings.setter
							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								raise NotImplementedError(
								    F'The model {self.model_type} is one of the few models that has no sequence length limit.'  )

