






import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    MODEL_FOR_PRETRAINING_MAPPING,
	    AlbertForMaskedLM,
	    AlbertForMultipleChoice,
	    AlbertForPreTraining,
	    AlbertForQuestionAnswering,
	    AlbertForSequenceClassification,
	    AlbertForTokenClassification,
	    AlbertModel,
	)
	from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST

class     _SCREAMING_SNAKE_CASE       :





	def __init__(		self				:					int ,  __lowerCamelCase				:					Union[str, Any] ,  __lowerCamelCase				:					Optional[Any]=13 ,  __lowerCamelCase				:					List[str]=7 ,  __lowerCamelCase				:					Optional[Any]=True ,  __lowerCamelCase				:					Dict=True ,  __lowerCamelCase				:					str=True ,  __lowerCamelCase				:					int=True ,  __lowerCamelCase				:					int=99 ,  __lowerCamelCase				:					int=16 ,  __lowerCamelCase				:					int=36 ,  __lowerCamelCase				:					Any=6 ,  __lowerCamelCase				:					int=6 ,  __lowerCamelCase				:					Dict=6 ,  __lowerCamelCase				:					Tuple=37 ,  __lowerCamelCase				:					Union[str, Any]="gelu" ,  __lowerCamelCase				:					Optional[Any]=0.1 ,  __lowerCamelCase				:					List[str]=0.1 ,  __lowerCamelCase				:					str=512 ,  __lowerCamelCase				:					List[Any]=16 ,  __lowerCamelCase				:					Optional[int]=2 ,  __lowerCamelCase				:					List[str]=0.02 ,  __lowerCamelCase				:					Union[str, Any]=3 ,  __lowerCamelCase				:					Union[str, Any]=4 ,  __lowerCamelCase				:					Optional[Any]=None ,  ):
		UpperCamelCase     :List[Any]          =	parent
		UpperCamelCase     :Optional[int]          =	batch_size
		UpperCamelCase     :Optional[Any]          =	seq_length
		UpperCamelCase     :Optional[int]          =	is_training
		UpperCamelCase     :str          =	use_input_mask
		UpperCamelCase     :List[str]          =	use_token_type_ids
		UpperCamelCase     :List[str]          =	use_labels
		UpperCamelCase     :Union[str, Any]          =	vocab_size
		UpperCamelCase     :Optional[Any]          =	embedding_size
		UpperCamelCase     :str          =	hidden_size
		UpperCamelCase     :Union[str, Any]          =	num_hidden_layers
		UpperCamelCase     :Union[str, Any]          =	num_hidden_groups
		UpperCamelCase     :Dict          =	num_attention_heads
		UpperCamelCase     :int          =	intermediate_size
		UpperCamelCase     :List[Any]          =	hidden_act
		UpperCamelCase     :Any          =	hidden_dropout_prob
		UpperCamelCase     :str          =	attention_probs_dropout_prob
		UpperCamelCase     :Optional[int]          =	max_position_embeddings
		UpperCamelCase     :Any          =	type_vocab_size
		UpperCamelCase     :List[str]          =	type_sequence_label_size
		UpperCamelCase     :int          =	initializer_range
		UpperCamelCase     :List[Any]          =	num_labels
		UpperCamelCase     :List[Any]          =	num_choices
		UpperCamelCase     :List[Any]          =	scope





	def 						_A (		self				:					int	):
		UpperCamelCase     :Optional[Any]          =	ids_tensor([self.batch_size, self.seq_length] ,  self.vocab_size	)

		UpperCamelCase     :Dict          =	None
		if self.use_input_mask:
			UpperCamelCase     :Dict          =	random_attention_mask([self.batch_size, self.seq_length]	)

		UpperCamelCase     :Optional[Any]          =	None
		if self.use_token_type_ids:
			UpperCamelCase     :List[str]          =	ids_tensor([self.batch_size, self.seq_length] ,  self.type_vocab_size	)

		UpperCamelCase     :str          =	None
		UpperCamelCase     :Union[str, Any]          =	None
		UpperCamelCase     :List[str]          =	None
		if self.use_labels:
			UpperCamelCase     :List[str]          =	ids_tensor([self.batch_size] ,  self.type_sequence_label_size	)
			UpperCamelCase     :Optional[Any]          =	ids_tensor([self.batch_size, self.seq_length] ,  self.num_labels	)
			UpperCamelCase     :int          =	ids_tensor([self.batch_size] ,  self.num_choices	)

		UpperCamelCase     :str          =	self.get_config()

		return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





	def 						_A (		self				:					int	):
		return AlbertConfig(
		    vocab_size=self.vocab_size ,  hidden_size=self.hidden_size ,  num_hidden_layers=self.num_hidden_layers ,  num_attention_heads=self.num_attention_heads ,  intermediate_size=self.intermediate_size ,  hidden_act=self.hidden_act ,  hidden_dropout_prob=self.hidden_dropout_prob ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob ,  max_position_embeddings=self.max_position_embeddings ,  type_vocab_size=self.type_vocab_size ,  initializer_range=self.initializer_range ,  num_hidden_groups=self.num_hidden_groups ,  )





	def 						_A (		self				:					Any ,  __lowerCamelCase				:					int ,  __lowerCamelCase				:					Tuple ,  __lowerCamelCase				:					Tuple ,  __lowerCamelCase				:					Dict ,  __lowerCamelCase				:					List[Any] ,  __lowerCamelCase				:					str ,  __lowerCamelCase				:					str	):
		UpperCamelCase     :Dict          =	AlbertModel(config=__lowerCamelCase	)
		model.to(__lowerCamelCase	)
		model.eval()
		UpperCamelCase     :List[Any]          =	model(__lowerCamelCase ,  attention_mask=__lowerCamelCase ,  token_type_ids=__lowerCamelCase	)
		UpperCamelCase     :str          =	model(__lowerCamelCase ,  token_type_ids=__lowerCamelCase	)
		UpperCamelCase     :Dict          =	model(__lowerCamelCase	)
		self.parent.assertEqual(result.last_hidden_state.shape ,  (self.batch_size, self.seq_length, self.hidden_size)	)
		self.parent.assertEqual(result.pooler_output.shape ,  (self.batch_size, self.hidden_size)	)





	def 						_A (		self				:					Optional[Any] ,  __lowerCamelCase				:					str ,  __lowerCamelCase				:					List[Any] ,  __lowerCamelCase				:					Tuple ,  __lowerCamelCase				:					Tuple ,  __lowerCamelCase				:					Optional[Any] ,  __lowerCamelCase				:					List[Any] ,  __lowerCamelCase				:					List[str]	):
		UpperCamelCase     :Tuple          =	AlbertForPreTraining(config=__lowerCamelCase	)
		model.to(__lowerCamelCase	)
		model.eval()
		UpperCamelCase     :Optional[Any]          =	model(
		    __lowerCamelCase ,  attention_mask=__lowerCamelCase ,  token_type_ids=__lowerCamelCase ,  labels=__lowerCamelCase ,  sentence_order_label=__lowerCamelCase ,  )
		self.parent.assertEqual(result.prediction_logits.shape ,  (self.batch_size, self.seq_length, self.vocab_size)	)
		self.parent.assertEqual(result.sop_logits.shape ,  (self.batch_size, config.num_labels)	)





	def 						_A (		self				:					Optional[Any] ,  __lowerCamelCase				:					List[Any] ,  __lowerCamelCase				:					Optional[int] ,  __lowerCamelCase				:					Union[str, Any] ,  __lowerCamelCase				:					Dict ,  __lowerCamelCase				:					Optional[int] ,  __lowerCamelCase				:					Any ,  __lowerCamelCase				:					int	):
		UpperCamelCase     :List[Any]          =	AlbertForMaskedLM(config=__lowerCamelCase	)
		model.to(__lowerCamelCase	)
		model.eval()
		UpperCamelCase     :Any          =	model(__lowerCamelCase ,  attention_mask=__lowerCamelCase ,  token_type_ids=__lowerCamelCase ,  labels=__lowerCamelCase	)
		self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.seq_length, self.vocab_size)	)





	def 						_A (		self				:					str ,  __lowerCamelCase				:					Union[str, Any] ,  __lowerCamelCase				:					str ,  __lowerCamelCase				:					Any ,  __lowerCamelCase				:					Tuple ,  __lowerCamelCase				:					List[Any] ,  __lowerCamelCase				:					Optional[Any] ,  __lowerCamelCase				:					str	):
		UpperCamelCase     :Optional[Any]          =	AlbertForQuestionAnswering(config=__lowerCamelCase	)
		model.to(__lowerCamelCase	)
		model.eval()
		UpperCamelCase     :List[str]          =	model(
		    __lowerCamelCase ,  attention_mask=__lowerCamelCase ,  token_type_ids=__lowerCamelCase ,  start_positions=__lowerCamelCase ,  end_positions=__lowerCamelCase ,  )
		self.parent.assertEqual(result.start_logits.shape ,  (self.batch_size, self.seq_length)	)
		self.parent.assertEqual(result.end_logits.shape ,  (self.batch_size, self.seq_length)	)





	def 						_A (		self				:					Tuple ,  __lowerCamelCase				:					Any ,  __lowerCamelCase				:					Union[str, Any] ,  __lowerCamelCase				:					Optional[Any] ,  __lowerCamelCase				:					Union[str, Any] ,  __lowerCamelCase				:					int ,  __lowerCamelCase				:					str ,  __lowerCamelCase				:					int	):
		UpperCamelCase     :int          =	self.num_labels
		UpperCamelCase     :int          =	AlbertForSequenceClassification(__lowerCamelCase	)
		model.to(__lowerCamelCase	)
		model.eval()
		UpperCamelCase     :Tuple          =	model(__lowerCamelCase ,  attention_mask=__lowerCamelCase ,  token_type_ids=__lowerCamelCase ,  labels=__lowerCamelCase	)
		self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.num_labels)	)





	def 						_A (		self				:					int ,  __lowerCamelCase				:					int ,  __lowerCamelCase				:					Union[str, Any] ,  __lowerCamelCase				:					Any ,  __lowerCamelCase				:					List[str] ,  __lowerCamelCase				:					Optional[Any] ,  __lowerCamelCase				:					Tuple ,  __lowerCamelCase				:					Dict	):
		UpperCamelCase     :Union[str, Any]          =	self.num_labels
		UpperCamelCase     :Any          =	AlbertForTokenClassification(config=__lowerCamelCase	)
		model.to(__lowerCamelCase	)
		model.eval()
		UpperCamelCase     :List[str]          =	model(__lowerCamelCase ,  attention_mask=__lowerCamelCase ,  token_type_ids=__lowerCamelCase ,  labels=__lowerCamelCase	)
		self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.seq_length, self.num_labels)	)





	def 						_A (		self				:					Optional[int] ,  __lowerCamelCase				:					List[str] ,  __lowerCamelCase				:					List[str] ,  __lowerCamelCase				:					List[str] ,  __lowerCamelCase				:					Optional[int] ,  __lowerCamelCase				:					Union[str, Any] ,  __lowerCamelCase				:					Optional[Any] ,  __lowerCamelCase				:					Tuple	):
		UpperCamelCase     :Dict          =	self.num_choices
		UpperCamelCase     :List[Any]          =	AlbertForMultipleChoice(config=__lowerCamelCase	)
		model.to(__lowerCamelCase	)
		model.eval()
		UpperCamelCase     :Any          =	input_ids.unsqueeze(1	).expand(-1 ,  self.num_choices ,  -1	).contiguous()
		UpperCamelCase     :Dict          =	token_type_ids.unsqueeze(1	).expand(-1 ,  self.num_choices ,  -1	).contiguous()
		UpperCamelCase     :List[Any]          =	input_mask.unsqueeze(1	).expand(-1 ,  self.num_choices ,  -1	).contiguous()
		UpperCamelCase     :int          =	model(
		    __lowerCamelCase ,  attention_mask=__lowerCamelCase ,  token_type_ids=__lowerCamelCase ,  labels=__lowerCamelCase ,  )
		self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.num_choices)	)





	def 						_A (		self				:					int	):
		UpperCamelCase     :Any          =	self.prepare_config_and_inputs()
		(
		    (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      
		)     :Optional[int]          =	config_and_inputs
		UpperCamelCase     :Optional[int]          =	{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
		return config, inputs_dict

@require_torch
class     _SCREAMING_SNAKE_CASE       (			_a		,   _a		,   unittest.TestCase      ):
	snake_case__							:     int     	=  (
	    (
	        AlbertModel,
	        AlbertForPreTraining,
	        AlbertForMaskedLM,
	        AlbertForMultipleChoice,
	        AlbertForSequenceClassification,
	        AlbertForTokenClassification,
	        AlbertForQuestionAnswering,
	    )
	    if is_torch_available()
	    else ()
	)
	snake_case__							:     List[Any]     	=  (
	    {
	        """feature-extraction""": AlbertModel,
	        """fill-mask""": AlbertForMaskedLM,
	        """question-answering""": AlbertForQuestionAnswering,
	        """text-classification""": AlbertForSequenceClassification,
	        """token-classification""": AlbertForTokenClassification,
	        """zero-shot""": AlbertForSequenceClassification,
	    }
	    if is_torch_available()
	    else {}
	)
	snake_case__							:     Dict     	=  True





	def 						_A (		self				:					List[Any] ,  __lowerCamelCase				:					List[str] ,  __lowerCamelCase				:					str ,  __lowerCamelCase				:					Dict=False	):
		UpperCamelCase     :Tuple          =	super()._prepare_for_class(__lowerCamelCase ,  __lowerCamelCase ,  return_labels=__lowerCamelCase	)

		if return_labels:
			if model_class in get_values(__lowerCamelCase	):
				UpperCamelCase     :Optional[int]          =	torch.zeros(
				    (self.model_tester.batch_size, self.model_tester.seq_length) ,  dtype=torch.long ,  device=__lowerCamelCase	)
				UpperCamelCase     :Optional[int]          =	torch.zeros(
				    self.model_tester.batch_size ,  dtype=torch.long ,  device=__lowerCamelCase	)
		return inputs_dict





	def 						_A (		self				:					Any	):
		UpperCamelCase     :Optional[int]          =	AlbertModelTester(self	)
		UpperCamelCase     :Tuple          =	ConfigTester(self ,  config_class=__lowerCamelCase ,  hidden_size=37	)





	def 						_A (		self				:					Dict	):
		self.config_tester.run_common_tests()





	def 						_A (		self				:					Dict	):
		UpperCamelCase     :int          =	self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_model(*__lowerCamelCase	)





	def 						_A (		self				:					Union[str, Any]	):
		UpperCamelCase     :Dict          =	self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_for_pretraining(*__lowerCamelCase	)





	def 						_A (		self				:					List[Any]	):
		UpperCamelCase     :Union[str, Any]          =	self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_for_masked_lm(*__lowerCamelCase	)





	def 						_A (		self				:					Optional[int]	):
		UpperCamelCase     :List[Any]          =	self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_for_multiple_choice(*__lowerCamelCase	)





	def 						_A (		self				:					str	):
		UpperCamelCase     :Any          =	self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_for_question_answering(*__lowerCamelCase	)





	def 						_A (		self				:					Dict	):
		UpperCamelCase     :Optional[int]          =	self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_for_sequence_classification(*__lowerCamelCase	)





	def 						_A (		self				:					Any	):
		UpperCamelCase     :Union[str, Any]          =	self.model_tester.prepare_config_and_inputs()
		for type in ["absolute", "relative_key", "relative_key_query"]:
			UpperCamelCase     :Optional[Any]          =	type
			self.model_tester.create_and_check_model(*__lowerCamelCase	)





	@slow
	def 						_A (		self				:					Dict	):
		for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
			UpperCamelCase     :List[Any]          =	AlbertModel.from_pretrained(__lowerCamelCase	)
			self.assertIsNotNone(__lowerCamelCase	)

@require_torch
class     _SCREAMING_SNAKE_CASE       (			unittest.TestCase      ):





	@slow
	def 						_A (		self				:					Union[str, Any]	):
		UpperCamelCase     :int          =	AlbertModel.from_pretrained("""albert-base-v2"""	)
		UpperCamelCase     :str          =	torch.tensor([[0, 345, 232, 328, 740, 140, 1_695, 69, 6_078, 1_588, 2]]	)
		UpperCamelCase     :Any          =	torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]	)
		with torch.no_grad():
			UpperCamelCase     :List[str]          =	model(__lowerCamelCase ,  attention_mask=__lowerCamelCase	)[0]
		UpperCamelCase     :Union[str, Any]          =	torch.Size((1, 11, 768)	)
		self.assertEqual(output.shape ,  __lowerCamelCase	)
		UpperCamelCase     :Union[str, Any]          =	torch.tensor(
		    [[[-0.6513, 1.5035, -0.2766], [-0.6515, 1.5046, -0.2780], [-0.6512, 1.5049, -0.2784]]]	)

		self.assertTrue(torch.allclose(output[:, 1:4, 1:4] ,  __lowerCamelCase ,  atol=1E-4	)	)








import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin

class     _SCREAMING_SNAKE_CASE       (			_a		,   unittest.TestCase      ):
	snake_case__							:     Optional[Any]     	=  TransfoXLTokenizer
	snake_case__							:     List[Any]     	=  False
	snake_case__							:     Tuple     	=  False





	def 						_A (		self				:					str	):
		super().setUp()

		UpperCamelCase     :Dict          =	[
		    """<unk>""",
		    """[CLS]""",
		    """[SEP]""",
		    """want""",
		    """unwanted""",
		    """wa""",
		    """un""",
		    """running""",
		    """,""",
		    """low""",
		    """l""",
		]
		UpperCamelCase     :str          =	os.path.join(self.tmpdirname ,  VOCAB_FILES_NAMES["""vocab_file"""]	)
		with open(self.vocab_file ,  """w""" ,  encoding="""utf-8"""	) as vocab_writer:
			vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]	)	)





	def 						_A (		self				:					List[str] ,  **__lowerCamelCase				:					Any	):
		UpperCamelCase     :Any          =	True
		return TransfoXLTokenizer.from_pretrained(self.tmpdirname ,  **__lowerCamelCase	)





	def 						_A (		self				:					Any ,  __lowerCamelCase				:					int	):
		UpperCamelCase     :List[Any]          =	"""<unk> UNwanted , running"""
		UpperCamelCase     :int          =	"""<unk> unwanted, running"""
		return input_text, output_text





	def 						_A (		self				:					Tuple	):
		UpperCamelCase     :List[str]          =	TransfoXLTokenizer(vocab_file=self.vocab_file ,  lower_case=__lowerCamelCase	)

		UpperCamelCase     :Any          =	tokenizer.tokenize("""<unk> UNwanted , running"""	)
		self.assertListEqual(__lowerCamelCase ,  ["""<unk>""", """unwanted""", """,""", """running"""]	)

		self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase	) ,  [0, 4, 8, 7]	)





	def 						_A (		self				:					Optional[Any]	):
		UpperCamelCase     :List[Any]          =	TransfoXLTokenizer(lower_case=__lowerCamelCase	)

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """	) ,  ["""hello""", """!""", """how""", """are""", """you""", """?"""]	)





	def 						_A (		self				:					Union[str, Any]	):
		UpperCamelCase     :int          =	TransfoXLTokenizer(lower_case=__lowerCamelCase	)

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """	) ,  ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]	)





	def 						_A (		self				:					Tuple	):
		UpperCamelCase     :Any          =	TransfoXLTokenizer(lower_case=__lowerCamelCase	)
		UpperCamelCase     :Optional[int]          =	"""Hello (bracket) and side-scrolled [and] Henry's $5,000 with 3.34 m. What's up!?"""
		UpperCamelCase     :Optional[int]          =	[
		    """Hello""",
		    """(""",
		    """bracket""",
		    """)""",
		    """and""",
		    """side""",
		    """@-@""",
		    """scrolled""",
		    """[""",
		    """and""",
		    """]""",
		    """Henry""",
		    """'s""",
		    """$""",
		    """5""",
		    """@,@""",
		    """000""",
		    """with""",
		    """3""",
		    """@.@""",
		    """34""",
		    """m""",
		    """.""",
		    """What""",
		    """'s""",
		    """up""",
		    """!""",
		    """?""",
		]

		self.assertListEqual(tokenizer.tokenize(__lowerCamelCase	) ,  __lowerCamelCase	)

		self.assertEqual(tokenizer.convert_tokens_to_string(__lowerCamelCase	) ,  __lowerCamelCase	)





	def 						_A (		self				:					List[Any]	):
		UpperCamelCase     :Any          =	self.get_tokenizer()
		UpperCamelCase     :List[str]          =	len(__lowerCamelCase	)

		tokenizer.add_tokens(["""new1""", """new2"""]	)
		tokenizer.move_added_token("""new1""" ,  1	)

		# Check that moved token is not copied (duplicate)
		self.assertEqual(len(__lowerCamelCase	) ,  original_len + 2	)
		# Check that token is moved to specified id
		self.assertEqual(tokenizer.encode("""new1"""	) ,  [1]	)
		self.assertEqual(tokenizer.decode([1]	) ,  """new1"""	)
