






import unittest

from parameterized import parameterized

from transformers import LlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import LlamaForCausalLM, LlamaForSequenceClassification, LlamaModel, LlamaTokenizer

class     _SCREAMING_SNAKE_CASE       :





	def __init__(		self				:					str ,  __lowerCamelCase				:					Optional[int] ,  __lowerCamelCase				:					Optional[int]=13 ,  __lowerCamelCase				:					List[str]=7 ,  __lowerCamelCase				:					List[str]=True ,  __lowerCamelCase				:					Optional[int]=True ,  __lowerCamelCase				:					int=False ,  __lowerCamelCase				:					Optional[Any]=True ,  __lowerCamelCase				:					Optional[int]=99 ,  __lowerCamelCase				:					Optional[int]=32 ,  __lowerCamelCase				:					Dict=5 ,  __lowerCamelCase				:					List[Any]=4 ,  __lowerCamelCase				:					List[str]=37 ,  __lowerCamelCase				:					Union[str, Any]="gelu" ,  __lowerCamelCase				:					Optional[int]=0.1 ,  __lowerCamelCase				:					Optional[int]=0.1 ,  __lowerCamelCase				:					Tuple=512 ,  __lowerCamelCase				:					str=16 ,  __lowerCamelCase				:					List[Any]=2 ,  __lowerCamelCase				:					List[str]=0.02 ,  __lowerCamelCase				:					Tuple=3 ,  __lowerCamelCase				:					str=4 ,  __lowerCamelCase				:					Union[str, Any]=None ,  ):
		UpperCamelCase     :Dict          =	parent
		UpperCamelCase     :List[Any]          =	batch_size
		UpperCamelCase     :List[str]          =	seq_length
		UpperCamelCase     :Optional[int]          =	is_training
		UpperCamelCase     :int          =	use_input_mask
		UpperCamelCase     :List[str]          =	use_token_type_ids
		UpperCamelCase     :int          =	use_labels
		UpperCamelCase     :int          =	vocab_size
		UpperCamelCase     :Optional[Any]          =	hidden_size
		UpperCamelCase     :Optional[Any]          =	num_hidden_layers
		UpperCamelCase     :Dict          =	num_attention_heads
		UpperCamelCase     :Optional[int]          =	intermediate_size
		UpperCamelCase     :int          =	hidden_act
		UpperCamelCase     :Dict          =	hidden_dropout_prob
		UpperCamelCase     :int          =	attention_probs_dropout_prob
		UpperCamelCase     :int          =	max_position_embeddings
		UpperCamelCase     :List[str]          =	type_vocab_size
		UpperCamelCase     :Union[str, Any]          =	type_sequence_label_size
		UpperCamelCase     :Dict          =	initializer_range
		UpperCamelCase     :Tuple          =	num_labels
		UpperCamelCase     :Optional[Any]          =	num_choices
		UpperCamelCase     :Optional[Any]          =	scope





	def 						_A (		self				:					Union[str, Any]	):
		UpperCamelCase     :Optional[Any]          =	ids_tensor([self.batch_size, self.seq_length] ,  self.vocab_size	)

		UpperCamelCase     :Optional[int]          =	None
		if self.use_input_mask:
			UpperCamelCase     :Tuple          =	random_attention_mask([self.batch_size, self.seq_length]	)

		UpperCamelCase     :Optional[Any]          =	None
		if self.use_token_type_ids:
			UpperCamelCase     :List[Any]          =	ids_tensor([self.batch_size, self.seq_length] ,  self.type_vocab_size	)

		UpperCamelCase     :str          =	None
		UpperCamelCase     :Optional[Any]          =	None
		UpperCamelCase     :List[Any]          =	None
		if self.use_labels:
			UpperCamelCase     :List[str]          =	ids_tensor([self.batch_size] ,  self.type_sequence_label_size	)
			UpperCamelCase     :Dict          =	ids_tensor([self.batch_size, self.seq_length] ,  self.num_labels	)
			UpperCamelCase     :int          =	ids_tensor([self.batch_size] ,  self.num_choices	)

		UpperCamelCase     :List[str]          =	self.get_config()

		return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





	def 						_A (		self				:					Optional[Any]	):
		return LlamaConfig(
		    vocab_size=self.vocab_size ,  hidden_size=self.hidden_size ,  num_hidden_layers=self.num_hidden_layers ,  num_attention_heads=self.num_attention_heads ,  intermediate_size=self.intermediate_size ,  hidden_act=self.hidden_act ,  hidden_dropout_prob=self.hidden_dropout_prob ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob ,  max_position_embeddings=self.max_position_embeddings ,  type_vocab_size=self.type_vocab_size ,  is_decoder=__lowerCamelCase ,  initializer_range=self.initializer_range ,  )





	def 						_A (		self				:					List[Any] ,  __lowerCamelCase				:					Union[str, Any] ,  __lowerCamelCase				:					Optional[int] ,  __lowerCamelCase				:					Optional[Any] ,  __lowerCamelCase				:					Optional[Any] ,  __lowerCamelCase				:					Optional[Any] ,  __lowerCamelCase				:					Dict ,  __lowerCamelCase				:					Optional[Any]	):
		UpperCamelCase     :Optional[Any]          =	LlamaModel(config=__lowerCamelCase	)
		model.to(__lowerCamelCase	)
		model.eval()
		UpperCamelCase     :List[Any]          =	model(__lowerCamelCase ,  attention_mask=__lowerCamelCase	)
		UpperCamelCase     :Optional[Any]          =	model(__lowerCamelCase	)
		self.parent.assertEqual(result.last_hidden_state.shape ,  (self.batch_size, self.seq_length, self.hidden_size)	)





	def 						_A (		self				:					Optional[int] ,  __lowerCamelCase				:					List[Any] ,  __lowerCamelCase				:					Union[str, Any] ,  __lowerCamelCase				:					Optional[Any] ,  __lowerCamelCase				:					Optional[Any] ,  __lowerCamelCase				:					Optional[Any] ,  __lowerCamelCase				:					Tuple ,  __lowerCamelCase				:					Dict ,  __lowerCamelCase				:					int ,  __lowerCamelCase				:					str ,  ):
		UpperCamelCase     :Optional[int]          =	True
		UpperCamelCase     :Any          =	LlamaModel(__lowerCamelCase	)
		model.to(__lowerCamelCase	)
		model.eval()
		UpperCamelCase     :List[Any]          =	model(
		    __lowerCamelCase ,  attention_mask=__lowerCamelCase ,  encoder_hidden_states=__lowerCamelCase ,  encoder_attention_mask=__lowerCamelCase ,  )
		UpperCamelCase     :Optional[Any]          =	model(
		    __lowerCamelCase ,  attention_mask=__lowerCamelCase ,  encoder_hidden_states=__lowerCamelCase ,  )
		UpperCamelCase     :Dict          =	model(__lowerCamelCase ,  attention_mask=__lowerCamelCase	)
		self.parent.assertEqual(result.last_hidden_state.shape ,  (self.batch_size, self.seq_length, self.hidden_size)	)





	def 						_A (		self				:					int ,  __lowerCamelCase				:					Union[str, Any] ,  __lowerCamelCase				:					str ,  __lowerCamelCase				:					int ,  __lowerCamelCase				:					Union[str, Any] ,  __lowerCamelCase				:					Tuple ,  __lowerCamelCase				:					Union[str, Any] ,  __lowerCamelCase				:					List[str] ,  __lowerCamelCase				:					Dict ,  __lowerCamelCase				:					str ,  ):
		UpperCamelCase     :str          =	LlamaForCausalLM(config=__lowerCamelCase	)
		model.to(__lowerCamelCase	)
		model.eval()
		UpperCamelCase     :List[str]          =	model(__lowerCamelCase ,  attention_mask=__lowerCamelCase ,  labels=__lowerCamelCase	)
		self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.seq_length, self.vocab_size)	)





	def 						_A (		self				:					str ,  __lowerCamelCase				:					Tuple ,  __lowerCamelCase				:					int ,  __lowerCamelCase				:					Optional[int] ,  __lowerCamelCase				:					Any ,  __lowerCamelCase				:					List[str] ,  __lowerCamelCase				:					Optional[int] ,  __lowerCamelCase				:					Tuple ,  __lowerCamelCase				:					Optional[int] ,  __lowerCamelCase				:					Tuple ,  ):
		UpperCamelCase     :str          =	True
		UpperCamelCase     :List[Any]          =	True
		UpperCamelCase     :Optional[Any]          =	LlamaForCausalLM(config=__lowerCamelCase	)
		model.to(__lowerCamelCase	)
		model.eval()

		# first forward pass
		UpperCamelCase     :Any          =	model(
		    __lowerCamelCase ,  attention_mask=__lowerCamelCase ,  encoder_hidden_states=__lowerCamelCase ,  encoder_attention_mask=__lowerCamelCase ,  use_cache=__lowerCamelCase ,  )
		UpperCamelCase     :Union[str, Any]          =	outputs.past_key_values

		# create hypothetical multiple next token and extent to next_input_ids
		UpperCamelCase     :str          =	ids_tensor((self.batch_size, 3) ,  config.vocab_size	)
		UpperCamelCase     :List[Any]          =	ids_tensor((self.batch_size, 3) ,  vocab_size=2	)

		# append to next input_ids and
		UpperCamelCase     :Tuple          =	torch.cat([input_ids, next_tokens] ,  dim=-1	)
		UpperCamelCase     :List[str]          =	torch.cat([input_mask, next_mask] ,  dim=-1	)

		UpperCamelCase     :Optional[Any]          =	model(
		    __lowerCamelCase ,  attention_mask=__lowerCamelCase ,  encoder_hidden_states=__lowerCamelCase ,  encoder_attention_mask=__lowerCamelCase ,  output_hidden_states=__lowerCamelCase ,  )["""hidden_states"""][0]
		UpperCamelCase     :Union[str, Any]          =	model(
		    __lowerCamelCase ,  attention_mask=__lowerCamelCase ,  encoder_hidden_states=__lowerCamelCase ,  encoder_attention_mask=__lowerCamelCase ,  past_key_values=__lowerCamelCase ,  output_hidden_states=__lowerCamelCase ,  )["""hidden_states"""][0]

		# select random slice
		UpperCamelCase     :int          =	ids_tensor((1,) ,  output_from_past.shape[-1]	).item()
		UpperCamelCase     :str          =	output_from_no_past[:, -3:, random_slice_idx].detach()
		UpperCamelCase     :List[Any]          =	output_from_past[:, :, random_slice_idx].detach()

		self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]	)

		# test that outputs are equal for slice
		self.parent.assertTrue(torch.allclose(__lowerCamelCase ,  __lowerCamelCase ,  atol=1E-3	)	)





	def 						_A (		self				:					Any	):
		UpperCamelCase     :Union[str, Any]          =	self.prepare_config_and_inputs()
		(
		    (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      
		)     :Tuple          =	config_and_inputs
		UpperCamelCase     :Optional[int]          =	{"""input_ids""": input_ids, """attention_mask""": input_mask}
		return config, inputs_dict

@require_torch
class     _SCREAMING_SNAKE_CASE       (			_a		,   _a		,   _a		,   unittest.TestCase      ):
	snake_case__							:     Optional[Any]     	=  (LlamaModel, LlamaForCausalLM, LlamaForSequenceClassification) if is_torch_available() else ()
	snake_case__							:     List[str]     	=  (LlamaForCausalLM,) if is_torch_available() else ()
	snake_case__							:     Any     	=  (
	    {
	        """feature-extraction""": LlamaModel,
	        """text-classification""": LlamaForSequenceClassification,
	        """text-generation""": LlamaForCausalLM,
	        """zero-shot""": LlamaForSequenceClassification,
	    }
	    if is_torch_available()
	    else {}
	)
	snake_case__							:     List[str]     	=  False
	snake_case__							:     Tuple     	=  False





	def 						_A (		self				:					str	):
		UpperCamelCase     :str          =	LlamaModelTester(self	)
		UpperCamelCase     :Optional[Any]          =	ConfigTester(self ,  config_class=__lowerCamelCase ,  hidden_size=37	)





	def 						_A (		self				:					Tuple	):
		self.config_tester.run_common_tests()





	def 						_A (		self				:					List[str]	):
		UpperCamelCase     :Dict          =	self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_model(*__lowerCamelCase	)





	def 						_A (		self				:					List[str]	):
		UpperCamelCase     :int          =	self.model_tester.prepare_config_and_inputs()
		for type in ["absolute", "relative_key", "relative_key_query"]:
			UpperCamelCase     :Any          =	type
			self.model_tester.create_and_check_model(*__lowerCamelCase	)





	def 						_A (		self				:					List[str]	):
		UpperCamelCase     ,      UpperCamelCase     :Tuple          =	self.model_tester.prepare_config_and_inputs_for_common()
		UpperCamelCase     :Union[str, Any]          =	3
		UpperCamelCase     :Dict          =	input_dict["""input_ids"""]
		UpperCamelCase     :List[Any]          =	input_ids.ne(1	).to(__lowerCamelCase	)
		UpperCamelCase     :List[str]          =	ids_tensor([self.model_tester.batch_size] ,  self.model_tester.type_sequence_label_size	)
		UpperCamelCase     :Dict          =	LlamaForSequenceClassification(__lowerCamelCase	)
		model.to(__lowerCamelCase	)
		model.eval()
		UpperCamelCase     :Any          =	model(__lowerCamelCase ,  attention_mask=__lowerCamelCase ,  labels=__lowerCamelCase	)
		self.assertEqual(result.logits.shape ,  (self.model_tester.batch_size, self.model_tester.num_labels)	)





	def 						_A (		self				:					Optional[Any]	):
		UpperCamelCase     ,      UpperCamelCase     :Dict          =	self.model_tester.prepare_config_and_inputs_for_common()
		UpperCamelCase     :List[str]          =	3
		UpperCamelCase     :Optional[int]          =	"""single_label_classification"""
		UpperCamelCase     :Optional[Any]          =	input_dict["""input_ids"""]
		UpperCamelCase     :Union[str, Any]          =	input_ids.ne(1	).to(__lowerCamelCase	)
		UpperCamelCase     :Optional[int]          =	ids_tensor([self.model_tester.batch_size] ,  self.model_tester.type_sequence_label_size	)
		UpperCamelCase     :Union[str, Any]          =	LlamaForSequenceClassification(__lowerCamelCase	)
		model.to(__lowerCamelCase	)
		model.eval()
		UpperCamelCase     :Union[str, Any]          =	model(__lowerCamelCase ,  attention_mask=__lowerCamelCase ,  labels=__lowerCamelCase	)
		self.assertEqual(result.logits.shape ,  (self.model_tester.batch_size, self.model_tester.num_labels)	)





	def 						_A (		self				:					List[str]	):
		UpperCamelCase     ,      UpperCamelCase     :Tuple          =	self.model_tester.prepare_config_and_inputs_for_common()
		UpperCamelCase     :Dict          =	3
		UpperCamelCase     :Optional[Any]          =	"""multi_label_classification"""
		UpperCamelCase     :List[str]          =	input_dict["""input_ids"""]
		UpperCamelCase     :str          =	input_ids.ne(1	).to(__lowerCamelCase	)
		UpperCamelCase     :Tuple          =	ids_tensor(
		    [self.model_tester.batch_size, config.num_labels] ,  self.model_tester.type_sequence_label_size	).to(torch.float	)
		UpperCamelCase     :Tuple          =	LlamaForSequenceClassification(__lowerCamelCase	)
		model.to(__lowerCamelCase	)
		model.eval()
		UpperCamelCase     :Optional[Any]          =	model(__lowerCamelCase ,  attention_mask=__lowerCamelCase ,  labels=__lowerCamelCase	)
		self.assertEqual(result.logits.shape ,  (self.model_tester.batch_size, self.model_tester.num_labels)	)





	@unittest.skip("""LLaMA buffers include complex numbers, which breaks this test"""	)
	def 						_A (		self				:					Union[str, Any]	):
		pass





	@parameterized.expand([("""linear""",), ("""dynamic""",)]	)
	def 						_A (		self				:					Optional[int] ,  __lowerCamelCase				:					Optional[int]	):
		UpperCamelCase     ,      UpperCamelCase     :List[Any]          =	self.model_tester.prepare_config_and_inputs_for_common()
		UpperCamelCase     :str          =	ids_tensor([1, 10] ,  config.vocab_size	)
		UpperCamelCase     :Optional[int]          =	ids_tensor([1, int(config.max_position_embeddings * 1.5	)] ,  config.vocab_size	)

		set_seed(42	)  # Fixed seed at init time so the two models get the same random weights
		UpperCamelCase     :Dict          =	LlamaModel(__lowerCamelCase	)
		original_model.to(__lowerCamelCase	)
		original_model.eval()
		UpperCamelCase     :Any          =	original_model(__lowerCamelCase	).last_hidden_state
		UpperCamelCase     :Union[str, Any]          =	original_model(__lowerCamelCase	).last_hidden_state

		set_seed(42	)  # Fixed seed at init time so the two models get the same random weights
		UpperCamelCase     :List[str]          =	{"""type""": scaling_type, """factor""": 10.0}
		UpperCamelCase     :List[str]          =	LlamaModel(__lowerCamelCase	)
		scaled_model.to(__lowerCamelCase	)
		scaled_model.eval()
		UpperCamelCase     :Union[str, Any]          =	scaled_model(__lowerCamelCase	).last_hidden_state
		UpperCamelCase     :List[str]          =	scaled_model(__lowerCamelCase	).last_hidden_state

		# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
		# maximum sequence length, so the outputs for the short input should match.
		if scaling_type == "dynamic":
			self.assertTrue(torch.allclose(__lowerCamelCase ,  __lowerCamelCase ,  atol=1E-5	)	)
		else:
			self.assertFalse(torch.allclose(__lowerCamelCase ,  __lowerCamelCase ,  atol=1E-5	)	)

		# The output should be different for long inputs
		self.assertFalse(torch.allclose(__lowerCamelCase ,  __lowerCamelCase ,  atol=1E-5	)	)

@require_torch
class     _SCREAMING_SNAKE_CASE       (			unittest.TestCase      ):





	@unittest.skip("""Logits are not exactly the same, once we fix the instabalities somehow, will update!"""	)
	@slow
	def 						_A (		self				:					List[str]	):
		UpperCamelCase     :Any          =	[1, 306, 4_658, 278, 6_593, 310, 2_834, 338]
		UpperCamelCase     :Tuple          =	LlamaForCausalLM.from_pretrained("""meta-llama/Llama-2-7b-hf""" ,  device_map="""auto"""	)
		UpperCamelCase     :str          =	model(torch.tensor([input_ids]	)	)
		# Expected mean on dim = -1
		UpperCamelCase     :Dict          =	torch.tensor([[-6.6550, -4.1227, -4.9859, -3.2406, 0.8262, -3.0033, 1.2964, -3.3699]]	)
		torch.testing.assert_close(out.mean(-1	) ,  __lowerCamelCase ,  atol=1E-2 ,  rtol=1E-2	)
		# slicing logits[0, 0, 0:30]
		# fmt: off
		UpperCamelCase     :str          =	torch.tensor([-12.8281, -7.4453, -0.4639, -8.0625, -7.2500, -8.0000, -6.4883, -7.7695, -7.8438, -7.0312, -6.2188, -7.1328, -1.8496, 1.9961, -8.6250, -6.7227, -12.8281, -6.9492, -7.0742, -7.7852, -7.5820, -7.9062, -6.9375, -7.9805, -8.3438, -8.1562, -8.0469, -7.6250, -7.7422, -7.3398,]	)
		# fmt: on
		torch.testing.assert_close(out[0, 0, :30] ,  __lowerCamelCase ,  atol=1E-5 ,  rtol=1E-5	)





	@unittest.skip("""Logits are not exactly the same, once we fix the instabalities somehow, will update!"""	)
	@slow
	def 						_A (		self				:					Tuple	):
		UpperCamelCase     :Dict          =	[1, 306, 4_658, 278, 6_593, 310, 2_834, 338]
		UpperCamelCase     :Tuple          =	LlamaForCausalLM.from_pretrained("""meta-llama/Llama-2-13b-hf""" ,  device_map="""auto"""	)
		UpperCamelCase     :List[Any]          =	model(torch.tensor(__lowerCamelCase	)	)
		# Expected mean on dim = -1
		UpperCamelCase     :Tuple          =	torch.tensor([[-2.0622, -1.2794, -1.1638, -0.9788, -1.4603, -1.0238, -1.7893, -1.4411]]	)
		torch.testing.assert_close(out.mean(-1	) ,  __lowerCamelCase ,  atol=1E-2 ,  rtol=1E-2	)
		# slicing logits[0, 0, 0:30]
		# fmt: off
		UpperCamelCase     :List[Any]          =	torch.tensor([-8.1406, -8.0547, 2.7461, -1.2344, -0.1448, -1.8262, -1.0020, -1.8154, -1.6895, -1.8516, -2.3574, -0.9277, 3.7598, 6.5742, -1.2998, -0.1177, -8.1406, -2.9688, -2.9199, -3.1699, -3.5254, -2.3555, -2.7988, -3.4141, -2.8262, -4.5195, -3.3379, -3.3164, -2.7832, -3.0273]	)
		# fmt: on
		torch.testing.assert_close(out[0, 0, :30] ,  __lowerCamelCase ,  atol=1E-5 ,  rtol=1E-5	)





	@unittest.skip("""Logits are not exactly the same, once we fix the instabalities somehow, will update!"""	)
	@slow
	def 						_A (		self				:					Optional[int]	):
		UpperCamelCase     :Union[str, Any]          =	[1, 306, 4_658, 278, 6_593, 310, 2_834, 338]
		UpperCamelCase     :Dict          =	LlamaForCausalLM.from_pretrained("""meta-llama/Llama-2-13b-chat-hf""" ,  device_map="""auto"""	)
		UpperCamelCase     :Union[str, Any]          =	model(torch.tensor(__lowerCamelCase	)	)
		# Expected mean on dim = -1
		UpperCamelCase     :Tuple          =	torch.tensor([[-0.8562, -1.8520, -0.7551, -0.4162, -1.5161, -1.2038, -2.4823, -2.3254]]	)
		torch.testing.assert_close(out.mean(-1	) ,  __lowerCamelCase ,  atol=1E-2 ,  rtol=1E-2	)
		# slicing logits[0, 0, 0:30]
		# fmt: off
		UpperCamelCase     :Optional[Any]          =	torch.tensor([-2.2227, 4.8828, 0.9023, -0.4578, -0.7871, -0.1033, -0.6221, -0.5786, -0.7803, -1.0674, -1.2920, -0.1570, 0.8008, 2.0723, -0.9497, 0.2771, -2.2227, -0.7612, -1.4346, -1.2061, -1.6426, -0.3000, -0.7139, -1.1934, -1.8691, -1.6973, -1.5947, -1.2705, -0.3523, -0.5513]	)
		# fmt: on
		torch.testing.assert_close(out.mean(-1	) ,  __lowerCamelCase ,  atol=1E-2 ,  rtol=1E-2	)





	@unittest.skip(
	    """Logits are not exactly the same, once we fix the instabalities somehow, will update! Also it is gonna be a `too_slow` test"""	)
	@slow
	def 						_A (		self				:					Union[str, Any]	):
		UpperCamelCase     :Dict          =	[1, 306, 4_658, 278, 6_593, 310, 2_834, 338]
		UpperCamelCase     :Union[str, Any]          =	LlamaForCausalLM.from_pretrained("""meta-llama/Llama-2-70b-hf""" ,  device_map="""auto"""	)
		UpperCamelCase     :Optional[Any]          =	model(torch.tensor(__lowerCamelCase	)	)

		UpperCamelCase     :Any          =	torch.tensor(
		    [[-4.2327, -3.3360, -4.6665, -4.7631, -1.8180, -3.4170, -1.4211, -3.1810]] ,  dtype=torch.floataa	)
		torch.testing.assert_close(out.mean(-1	) ,  __lowerCamelCase ,  atol=1E-2 ,  rtol=1E-2	)
		# fmt: off
		UpperCamelCase     :str          =	torch.tensor([-9.4922, -3.9551, 1.7998, -5.6758, -5.1055, -5.8984, -4.8320, -6.8086, -6.5391, -5.6172, -5.5820, -5.5352, 1.7881, 3.6289, -6.5117, -3.4785, -9.5000, -6.0352, -6.8125, -6.0195, -6.6836, -5.4727, -6.2812, -6.0391, -7.3398, -7.4297, -7.4844, -6.5820, -5.8789, -5.5312]	)
		# fmt: on
		torch.testing.assert_close(out[0, 0, :30] ,  __lowerCamelCase ,  atol=1E-5 ,  rtol=1E-5	)





	@unittest.skip("""Model is curently gated"""	)
	@slow
	def 						_A (		self				:					Optional[int]	):
		UpperCamelCase     :Optional[Any]          =	"""Simply put, the theory of relativity states that 1) the laws of physics are the same everywhere in the universe and 2) the passage of time and the length of objects can vary depending on the observer\'s frame of reference.\n\nThe first part of the theory, that the laws of physics are the same everywhere, is known as the \"princi"""
		UpperCamelCase     :str          =	"""Simply put, the theory of relativity states that """
		UpperCamelCase     :Optional[int]          =	LlamaTokenizer.from_pretrained("""meta-llama/Llama-2-13b-chat-hf"""	)
		UpperCamelCase     :Union[str, Any]          =	tokenizer.encode(__lowerCamelCase ,  return_tensors="""pt"""	)
		UpperCamelCase     :int          =	LlamaForCausalLM.from_pretrained(
		    """meta-llama/Llama-2-13b-chat-hf""" ,  device_map="""sequential""" ,  use_safetensors=__lowerCamelCase	)

		# greedy generation outputs
		UpperCamelCase     :Any          =	model.generate(__lowerCamelCase ,  max_new_tokens=64 ,  top_p=__lowerCamelCase ,  temperature=1 ,  do_sample=__lowerCamelCase	)
		UpperCamelCase     :List[Any]          =	tokenizer.decode(generated_ids[0] ,  skip_special_tokens=__lowerCamelCase	)
		self.assertEqual(__lowerCamelCase ,  __lowerCamelCase	)








import os
import tempfile
import unittest

from transformers import FlaubertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    FlaubertForMultipleChoice,
	    FlaubertForQuestionAnswering,
	    FlaubertForQuestionAnsweringSimple,
	    FlaubertForSequenceClassification,
	    FlaubertForTokenClassification,
	    FlaubertModel,
	    FlaubertWithLMHeadModel,
	)
	from transformers.models.flaubert.modeling_flaubert import FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST

class     _SCREAMING_SNAKE_CASE       (			_a      ):





	def __init__(		self				:					List[str] ,  __lowerCamelCase				:					List[str] ,  __lowerCamelCase				:					Union[str, Any]=13 ,  __lowerCamelCase				:					str=7 ,  __lowerCamelCase				:					Tuple=True ,  __lowerCamelCase				:					Union[str, Any]=True ,  __lowerCamelCase				:					Any=True ,  __lowerCamelCase				:					List[Any]=True ,  __lowerCamelCase				:					Optional[Any]=True ,  __lowerCamelCase				:					Optional[int]=False ,  __lowerCamelCase				:					str=False ,  __lowerCamelCase				:					List[Any]=False ,  __lowerCamelCase				:					List[Any]=2 ,  __lowerCamelCase				:					Union[str, Any]=99 ,  __lowerCamelCase				:					Optional[Any]=0 ,  __lowerCamelCase				:					Tuple=32 ,  __lowerCamelCase				:					Any=5 ,  __lowerCamelCase				:					Optional[Any]=4 ,  __lowerCamelCase				:					Union[str, Any]=0.1 ,  __lowerCamelCase				:					Union[str, Any]=0.1 ,  __lowerCamelCase				:					List[str]=512 ,  __lowerCamelCase				:					List[Any]=12 ,  __lowerCamelCase				:					int=2 ,  __lowerCamelCase				:					List[str]=0.02 ,  __lowerCamelCase				:					Union[str, Any]=3 ,  __lowerCamelCase				:					Tuple=4 ,  __lowerCamelCase				:					Optional[int]="last" ,  __lowerCamelCase				:					Optional[Any]=None ,  __lowerCamelCase				:					List[str]=None ,  ):
		UpperCamelCase     :int          =	parent
		UpperCamelCase     :Optional[int]          =	batch_size
		UpperCamelCase     :str          =	seq_length
		UpperCamelCase     :Optional[int]          =	is_training
		UpperCamelCase     :Optional[int]          =	use_input_lengths
		UpperCamelCase     :Union[str, Any]          =	use_token_type_ids
		UpperCamelCase     :List[str]          =	use_labels
		UpperCamelCase     :Dict          =	gelu_activation
		UpperCamelCase     :Optional[int]          =	sinusoidal_embeddings
		UpperCamelCase     :List[Any]          =	causal
		UpperCamelCase     :Optional[int]          =	asm
		UpperCamelCase     :List[str]          =	n_langs
		UpperCamelCase     :int          =	vocab_size
		UpperCamelCase     :List[Any]          =	n_special
		UpperCamelCase     :List[Any]          =	hidden_size
		UpperCamelCase     :List[str]          =	num_hidden_layers
		UpperCamelCase     :List[Any]          =	num_attention_heads
		UpperCamelCase     :Tuple          =	hidden_dropout_prob
		UpperCamelCase     :List[str]          =	attention_probs_dropout_prob
		UpperCamelCase     :Tuple          =	max_position_embeddings
		UpperCamelCase     :List[str]          =	type_vocab_size
		UpperCamelCase     :Union[str, Any]          =	type_sequence_label_size
		UpperCamelCase     :int          =	initializer_range
		UpperCamelCase     :List[str]          =	num_labels
		UpperCamelCase     :Optional[int]          =	num_choices
		UpperCamelCase     :Optional[Any]          =	summary_type
		UpperCamelCase     :Tuple          =	use_proj
		UpperCamelCase     :Optional[Any]          =	scope





	def 						_A (		self				:					List[str]	):
		UpperCamelCase     :Union[str, Any]          =	ids_tensor([self.batch_size, self.seq_length] ,  self.vocab_size	)
		UpperCamelCase     :Any          =	random_attention_mask([self.batch_size, self.seq_length]	)

		UpperCamelCase     :List[Any]          =	None
		if self.use_input_lengths:
			UpperCamelCase     :Dict          =	(
			    ids_tensor([self.batch_size] ,  vocab_size=2	) + self.seq_length - 2
			)  # small variation of seq_length

		UpperCamelCase     :str          =	None
		if self.use_token_type_ids:
			UpperCamelCase     :int          =	ids_tensor([self.batch_size, self.seq_length] ,  self.n_langs	)

		UpperCamelCase     :Optional[int]          =	None
		UpperCamelCase     :int          =	None
		UpperCamelCase     :List[Any]          =	None
		if self.use_labels:
			UpperCamelCase     :Optional[Any]          =	ids_tensor([self.batch_size] ,  self.type_sequence_label_size	)
			UpperCamelCase     :List[str]          =	ids_tensor([self.batch_size, self.seq_length] ,  self.num_labels	)
			UpperCamelCase     :List[str]          =	ids_tensor([self.batch_size] ,  2	).float()
			UpperCamelCase     :List[str]          =	ids_tensor([self.batch_size] ,  self.num_choices	)

		UpperCamelCase     :Union[str, Any]          =	self.get_config()

		return (
		    config,
		    input_ids,
		    token_type_ids,
		    input_lengths,
		    sequence_labels,
		    token_labels,
		    is_impossible_labels,
		    choice_labels,
		    input_mask,
		)





	def 						_A (		self				:					List[Any]	):
		return FlaubertConfig(
		    vocab_size=self.vocab_size ,  n_special=self.n_special ,  emb_dim=self.hidden_size ,  n_layers=self.num_hidden_layers ,  n_heads=self.num_attention_heads ,  dropout=self.hidden_dropout_prob ,  attention_dropout=self.attention_probs_dropout_prob ,  gelu_activation=self.gelu_activation ,  sinusoidal_embeddings=self.sinusoidal_embeddings ,  asm=self.asm ,  causal=self.causal ,  n_langs=self.n_langs ,  max_position_embeddings=self.max_position_embeddings ,  initializer_range=self.initializer_range ,  summary_type=self.summary_type ,  use_proj=self.use_proj ,  )





	def 						_A (		self				:					Union[str, Any] ,  __lowerCamelCase				:					Optional[int] ,  __lowerCamelCase				:					Optional[Any] ,  __lowerCamelCase				:					int ,  __lowerCamelCase				:					Tuple ,  __lowerCamelCase				:					List[Any] ,  __lowerCamelCase				:					Optional[int] ,  __lowerCamelCase				:					List[Any] ,  __lowerCamelCase				:					Any ,  __lowerCamelCase				:					int ,  ):
		UpperCamelCase     :Tuple          =	FlaubertModel(config=__lowerCamelCase	)
		model.to(__lowerCamelCase	)
		model.eval()
		UpperCamelCase     :int          =	model(__lowerCamelCase ,  lengths=__lowerCamelCase ,  langs=__lowerCamelCase	)
		UpperCamelCase     :List[Any]          =	model(__lowerCamelCase ,  langs=__lowerCamelCase	)
		UpperCamelCase     :int          =	model(__lowerCamelCase	)
		self.parent.assertEqual(result.last_hidden_state.shape ,  (self.batch_size, self.seq_length, self.hidden_size)	)





	def 						_A (		self				:					Optional[int] ,  __lowerCamelCase				:					List[str] ,  __lowerCamelCase				:					Any ,  __lowerCamelCase				:					Tuple ,  __lowerCamelCase				:					int ,  __lowerCamelCase				:					List[Any] ,  __lowerCamelCase				:					Tuple ,  __lowerCamelCase				:					List[Any] ,  __lowerCamelCase				:					Any ,  __lowerCamelCase				:					Dict ,  ):
		UpperCamelCase     :Any          =	FlaubertWithLMHeadModel(__lowerCamelCase	)
		model.to(__lowerCamelCase	)
		model.eval()

		UpperCamelCase     :Dict          =	model(__lowerCamelCase ,  token_type_ids=__lowerCamelCase ,  labels=__lowerCamelCase	)
		self.parent.assertEqual(result.loss.shape ,  ()	)
		self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.seq_length, self.vocab_size)	)





	def 						_A (		self				:					List[Any] ,  __lowerCamelCase				:					str ,  __lowerCamelCase				:					Tuple ,  __lowerCamelCase				:					List[str] ,  __lowerCamelCase				:					int ,  __lowerCamelCase				:					Optional[Any] ,  __lowerCamelCase				:					Union[str, Any] ,  __lowerCamelCase				:					str ,  __lowerCamelCase				:					List[str] ,  __lowerCamelCase				:					List[Any] ,  ):
		UpperCamelCase     :Any          =	FlaubertForQuestionAnsweringSimple(__lowerCamelCase	)
		model.to(__lowerCamelCase	)
		model.eval()

		UpperCamelCase     :Any          =	model(__lowerCamelCase	)

		UpperCamelCase     :int          =	model(__lowerCamelCase ,  start_positions=__lowerCamelCase ,  end_positions=__lowerCamelCase	)
		self.parent.assertEqual(result.start_logits.shape ,  (self.batch_size, self.seq_length)	)
		self.parent.assertEqual(result.end_logits.shape ,  (self.batch_size, self.seq_length)	)





	def 						_A (		self				:					str ,  __lowerCamelCase				:					int ,  __lowerCamelCase				:					str ,  __lowerCamelCase				:					Optional[Any] ,  __lowerCamelCase				:					int ,  __lowerCamelCase				:					Tuple ,  __lowerCamelCase				:					Any ,  __lowerCamelCase				:					List[str] ,  __lowerCamelCase				:					Dict ,  __lowerCamelCase				:					str ,  ):
		UpperCamelCase     :str          =	FlaubertForQuestionAnswering(__lowerCamelCase	)
		model.to(__lowerCamelCase	)
		model.eval()

		UpperCamelCase     :Any          =	model(__lowerCamelCase	)

		UpperCamelCase     :Optional[int]          =	model(
		    __lowerCamelCase ,  start_positions=__lowerCamelCase ,  end_positions=__lowerCamelCase ,  cls_index=__lowerCamelCase ,  is_impossible=__lowerCamelCase ,  p_mask=__lowerCamelCase ,  )

		UpperCamelCase     :Union[str, Any]          =	model(
		    __lowerCamelCase ,  start_positions=__lowerCamelCase ,  end_positions=__lowerCamelCase ,  cls_index=__lowerCamelCase ,  is_impossible=__lowerCamelCase ,  )

		((UpperCamelCase)     ,      )     :int          =	result_with_labels.to_tuple()

		UpperCamelCase     :int          =	model(__lowerCamelCase ,  start_positions=__lowerCamelCase ,  end_positions=__lowerCamelCase	)

		((UpperCamelCase)     ,      )     :List[Any]          =	result_with_labels.to_tuple()

		self.parent.assertEqual(result_with_labels.loss.shape ,  ()	)
		self.parent.assertEqual(result.start_top_log_probs.shape ,  (self.batch_size, model.config.start_n_top)	)
		self.parent.assertEqual(result.start_top_index.shape ,  (self.batch_size, model.config.start_n_top)	)
		self.parent.assertEqual(
		    result.end_top_log_probs.shape ,  (self.batch_size, model.config.start_n_top * model.config.end_n_top)	)
		self.parent.assertEqual(
		    result.end_top_index.shape ,  (self.batch_size, model.config.start_n_top * model.config.end_n_top)	)
		self.parent.assertEqual(result.cls_logits.shape ,  (self.batch_size,)	)





	def 						_A (		self				:					List[Any] ,  __lowerCamelCase				:					Any ,  __lowerCamelCase				:					Any ,  __lowerCamelCase				:					int ,  __lowerCamelCase				:					Dict ,  __lowerCamelCase				:					Any ,  __lowerCamelCase				:					Any ,  __lowerCamelCase				:					Optional[int] ,  __lowerCamelCase				:					Optional[Any] ,  __lowerCamelCase				:					Tuple ,  ):
		UpperCamelCase     :Optional[int]          =	FlaubertForSequenceClassification(__lowerCamelCase	)
		model.to(__lowerCamelCase	)
		model.eval()

		UpperCamelCase     :Tuple          =	model(__lowerCamelCase	)
		UpperCamelCase     :List[str]          =	model(__lowerCamelCase ,  labels=__lowerCamelCase	)

		self.parent.assertEqual(result.loss.shape ,  ()	)
		self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.type_sequence_label_size)	)





	def 						_A (		self				:					Any ,  __lowerCamelCase				:					int ,  __lowerCamelCase				:					Optional[int] ,  __lowerCamelCase				:					str ,  __lowerCamelCase				:					Union[str, Any] ,  __lowerCamelCase				:					int ,  __lowerCamelCase				:					Optional[int] ,  __lowerCamelCase				:					Tuple ,  __lowerCamelCase				:					List[Any] ,  __lowerCamelCase				:					List[Any] ,  ):
		UpperCamelCase     :Dict          =	self.num_labels
		UpperCamelCase     :Tuple          =	FlaubertForTokenClassification(__lowerCamelCase	)
		model.to(__lowerCamelCase	)
		model.eval()

		UpperCamelCase     :Optional[Any]          =	model(__lowerCamelCase ,  attention_mask=__lowerCamelCase ,  labels=__lowerCamelCase	)
		self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.seq_length, self.num_labels)	)





	def 						_A (		self				:					Tuple ,  __lowerCamelCase				:					Tuple ,  __lowerCamelCase				:					Optional[int] ,  __lowerCamelCase				:					Optional[int] ,  __lowerCamelCase				:					Optional[Any] ,  __lowerCamelCase				:					List[str] ,  __lowerCamelCase				:					Tuple ,  __lowerCamelCase				:					Tuple ,  __lowerCamelCase				:					List[str] ,  __lowerCamelCase				:					List[Any] ,  ):
		UpperCamelCase     :Union[str, Any]          =	self.num_choices
		UpperCamelCase     :List[Any]          =	FlaubertForMultipleChoice(config=__lowerCamelCase	)
		model.to(__lowerCamelCase	)
		model.eval()
		UpperCamelCase     :Optional[Any]          =	input_ids.unsqueeze(1	).expand(-1 ,  self.num_choices ,  -1	).contiguous()
		UpperCamelCase     :Optional[int]          =	token_type_ids.unsqueeze(1	).expand(-1 ,  self.num_choices ,  -1	).contiguous()
		UpperCamelCase     :int          =	input_mask.unsqueeze(1	).expand(-1 ,  self.num_choices ,  -1	).contiguous()
		UpperCamelCase     :Union[str, Any]          =	model(
		    __lowerCamelCase ,  attention_mask=__lowerCamelCase ,  token_type_ids=__lowerCamelCase ,  labels=__lowerCamelCase ,  )
		self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.num_choices)	)





	def 						_A (		self				:					str	):
		UpperCamelCase     :List[str]          =	self.prepare_config_and_inputs()
		(
		    (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      
		)     :List[Any]          =	config_and_inputs
		UpperCamelCase     :Union[str, Any]          =	{
		    """input_ids""": input_ids,
		    """token_type_ids""": token_type_ids,
		    """lengths""": input_lengths,
		    """attention_mask""": input_mask,
		}
		return config, inputs_dict

@require_torch
class     _SCREAMING_SNAKE_CASE       (			_a		,   _a		,   unittest.TestCase      ):
	snake_case__							:     Optional[int]     	=  (
	    (
	        FlaubertModel,
	        FlaubertWithLMHeadModel,
	        FlaubertForQuestionAnswering,
	        FlaubertForQuestionAnsweringSimple,
	        FlaubertForSequenceClassification,
	        FlaubertForTokenClassification,
	        FlaubertForMultipleChoice,
	    )
	    if is_torch_available()
	    else ()
	)
	snake_case__							:     Tuple     	=  (
	    {
	        """feature-extraction""": FlaubertModel,
	        """fill-mask""": FlaubertWithLMHeadModel,
	        """question-answering""": FlaubertForQuestionAnsweringSimple,
	        """text-classification""": FlaubertForSequenceClassification,
	        """token-classification""": FlaubertForTokenClassification,
	        """zero-shot""": FlaubertForSequenceClassification,
	    }
	    if is_torch_available()
	    else {}
	)





	def 						_A (		self				:					int ,  __lowerCamelCase				:					List[str] ,  __lowerCamelCase				:					Tuple ,  __lowerCamelCase				:					List[str] ,  __lowerCamelCase				:					List[Any] ,  __lowerCamelCase				:					Optional[int]	):
		if (
		    pipeline_test_casse_name == "QAPipelineTests"
		    and tokenizer_name is not None
		    and not tokenizer_name.endswith("""Fast"""	)
		):
			# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
			# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
			# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
			return True

		return False





	def 						_A (		self				:					Optional[int] ,  __lowerCamelCase				:					Dict ,  __lowerCamelCase				:					Tuple ,  __lowerCamelCase				:					Tuple=False	):
		UpperCamelCase     :Tuple          =	super()._prepare_for_class(__lowerCamelCase ,  __lowerCamelCase ,  return_labels=__lowerCamelCase	)

		if return_labels:
			if model_class.__name__ == "FlaubertForQuestionAnswering":
				UpperCamelCase     :Tuple          =	torch.zeros(
				    self.model_tester.batch_size ,  dtype=torch.long ,  device=__lowerCamelCase	)
				UpperCamelCase     :List[Any]          =	torch.zeros(
				    self.model_tester.batch_size ,  dtype=torch.long ,  device=__lowerCamelCase	)

		return inputs_dict





	def 						_A (		self				:					str	):
		UpperCamelCase     :List[Any]          =	FlaubertModelTester(self	)
		UpperCamelCase     :Any          =	ConfigTester(self ,  config_class=__lowerCamelCase ,  emb_dim=37	)





	def 						_A (		self				:					Optional[int]	):
		self.config_tester.run_common_tests()





	def 						_A (		self				:					List[Any]	):
		UpperCamelCase     :List[Any]          =	self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_flaubert_model(*__lowerCamelCase	)





	def 						_A (		self				:					Optional[int]	):
		UpperCamelCase     :Optional[Any]          =	self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_flaubert_lm_head(*__lowerCamelCase	)





	def 						_A (		self				:					List[Any]	):
		UpperCamelCase     :Optional[int]          =	self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_flaubert_simple_qa(*__lowerCamelCase	)





	def 						_A (		self				:					Union[str, Any]	):
		UpperCamelCase     :int          =	self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_flaubert_qa(*__lowerCamelCase	)





	def 						_A (		self				:					Optional[Any]	):
		UpperCamelCase     :List[Any]          =	self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_flaubert_sequence_classif(*__lowerCamelCase	)





	def 						_A (		self				:					Tuple	):
		UpperCamelCase     :Tuple          =	self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_flaubert_token_classif(*__lowerCamelCase	)





	def 						_A (		self				:					int	):
		UpperCamelCase     :Tuple          =	self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_flaubert_multiple_choice(*__lowerCamelCase	)





	@slow
	def 						_A (		self				:					Any	):
		for model_name in FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
			UpperCamelCase     :Optional[int]          =	FlaubertModel.from_pretrained(__lowerCamelCase	)
			self.assertIsNotNone(__lowerCamelCase	)





	@slow
	@require_torch_gpu
	def 						_A (		self				:					Tuple	):
		UpperCamelCase     ,      UpperCamelCase     :Union[str, Any]          =	self.model_tester.prepare_config_and_inputs_for_common()
		for model_class in self.all_model_classes:
			# FlauBertForMultipleChoice behaves incorrectly in JIT environments.
			if model_class == FlaubertForMultipleChoice:
				return

			UpperCamelCase     :Optional[Any]          =	True
			UpperCamelCase     :Optional[Any]          =	model_class(config=__lowerCamelCase	)

			UpperCamelCase     :str          =	self._prepare_for_class(__lowerCamelCase ,  __lowerCamelCase	)
			UpperCamelCase     :str          =	torch.jit.trace(
			    __lowerCamelCase ,  (inputs_dict["""input_ids"""].to("""cpu"""	), inputs_dict["""attention_mask"""].to("""cpu"""	))	)

			with tempfile.TemporaryDirectory() as tmp:
				torch.jit.save(__lowerCamelCase ,  os.path.join(__lowerCamelCase ,  """traced_model.pt"""	)	)
				UpperCamelCase     :int          =	torch.jit.load(os.path.join(__lowerCamelCase ,  """traced_model.pt"""	) ,  map_location=__lowerCamelCase	)
				loaded(inputs_dict["""input_ids"""].to(__lowerCamelCase	) ,  inputs_dict["""attention_mask"""].to(__lowerCamelCase	)	)

@require_torch
class     _SCREAMING_SNAKE_CASE       (			unittest.TestCase      ):





	@slow
	def 						_A (		self				:					Optional[Any]	):
		UpperCamelCase     :Union[str, Any]          =	FlaubertModel.from_pretrained("""flaubert/flaubert_base_cased"""	)
		UpperCamelCase     :Optional[Any]          =	torch.tensor([[0, 345, 232, 328, 740, 140, 1_695, 69, 6_078, 1_588, 2]]	)
		with torch.no_grad():
			UpperCamelCase     :Tuple          =	model(__lowerCamelCase	)[0]
		UpperCamelCase     :Union[str, Any]          =	torch.Size((1, 11, 768)	)
		self.assertEqual(output.shape ,  __lowerCamelCase	)
		UpperCamelCase     :int          =	torch.tensor(
		    [[[-2.6251, -1.4298, -0.0227], [-2.8510, -1.6387, 0.2258], [-2.8114, -1.1832, -0.3066]]]	)

		self.assertTrue(torch.allclose(output[:, :3, :3] ,  __lowerCamelCase ,  atol=1E-4	)	)
