
import unittest

from parameterized import parameterized

from transformers import LlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import LlamaForCausalLM, LlamaForSequenceClassification, LlamaModel, LlamaTokenizer
class  lowercase__:



	"""simple docstring"""




	def __init__(       self  :			int       ,							SCREAMING_SNAKE_CASE_  :			Optional[int]       ,							SCREAMING_SNAKE_CASE_  :			List[Any]=1_3       ,							SCREAMING_SNAKE_CASE_  :			Dict=7       ,							SCREAMING_SNAKE_CASE_  :			str=True       ,							SCREAMING_SNAKE_CASE_  :			str=True       ,							SCREAMING_SNAKE_CASE_  :			Optional[int]=False       ,							SCREAMING_SNAKE_CASE_  :			Optional[int]=True       ,							SCREAMING_SNAKE_CASE_  :			Union[str, Any]=9_9       ,							SCREAMING_SNAKE_CASE_  :			Optional[int]=3_2       ,							SCREAMING_SNAKE_CASE_  :			Optional[int]=5       ,							SCREAMING_SNAKE_CASE_  :			Optional[Any]=4       ,							SCREAMING_SNAKE_CASE_  :			str=3_7       ,							SCREAMING_SNAKE_CASE_  :			Union[str, Any]="gelu"       ,							SCREAMING_SNAKE_CASE_  :			int=0.1       ,							SCREAMING_SNAKE_CASE_  :			List[Any]=0.1       ,							SCREAMING_SNAKE_CASE_  :			Dict=5_1_2       ,							SCREAMING_SNAKE_CASE_  :			Optional[int]=1_6       ,							SCREAMING_SNAKE_CASE_  :			List[Any]=2       ,							SCREAMING_SNAKE_CASE_  :			str=0.02       ,							SCREAMING_SNAKE_CASE_  :			int=3       ,							SCREAMING_SNAKE_CASE_  :			Tuple=4       ,							SCREAMING_SNAKE_CASE_  :			Optional[Any]=None       ,							)       ->     List[str]:
					lowercase_       =					parent
					lowercase_       =					batch_size
					lowercase_       =					seq_length
					lowercase_       =					is_training
					lowercase_       =					use_input_mask
					lowercase_       =					use_token_type_ids
					lowercase_       =					use_labels
					lowercase_       =					vocab_size
					lowercase_       =					hidden_size
					lowercase_       =					num_hidden_layers
					lowercase_       =					num_attention_heads
					lowercase_       =					intermediate_size
					lowercase_       =					hidden_act
					lowercase_       =					hidden_dropout_prob
					lowercase_       =					attention_probs_dropout_prob
					lowercase_       =					max_position_embeddings
					lowercase_       =					type_vocab_size
					lowercase_       =					type_sequence_label_size
					lowercase_       =					initializer_range
					lowercase_       =					num_labels
					lowercase_       =					num_choices
					lowercase_       =					scope
	def 						_lowercase  (       self  :			List[Any]       )       ->     Optional[int]:
					lowercase_       =					ids_tensor([self.batch_size, self.seq_length]       ,							self.vocab_size       )

					lowercase_       =					None
					if self.use_input_mask:
									lowercase_       =					random_attention_mask([self.batch_size, self.seq_length]       )

					lowercase_       =					None
					if self.use_token_type_ids:
									lowercase_       =					ids_tensor([self.batch_size, self.seq_length]       ,							self.type_vocab_size       )

					lowercase_       =					None
					lowercase_       =					None
					lowercase_       =					None
					if self.use_labels:
									lowercase_       =					ids_tensor([self.batch_size]       ,							self.type_sequence_label_size       )
									lowercase_       =					ids_tensor([self.batch_size, self.seq_length]       ,							self.num_labels       )
									lowercase_       =					ids_tensor([self.batch_size]       ,							self.num_choices       )

					lowercase_       =					self.get_config()

					return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
	def 						_lowercase  (       self  :			str       )       ->     List[str]:
					return LlamaConfig(
					    vocab_size=self.vocab_size       ,							hidden_size=self.hidden_size       ,							num_hidden_layers=self.num_hidden_layers       ,							num_attention_heads=self.num_attention_heads       ,							intermediate_size=self.intermediate_size       ,							hidden_act=self.hidden_act       ,							hidden_dropout_prob=self.hidden_dropout_prob       ,							attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,							max_position_embeddings=self.max_position_embeddings       ,							type_vocab_size=self.type_vocab_size       ,							is_decoder=a__       ,							initializer_range=self.initializer_range       ,							)
	def 						_lowercase  (       self  :			str       ,							SCREAMING_SNAKE_CASE_  :			int       ,							SCREAMING_SNAKE_CASE_  :			str       ,							SCREAMING_SNAKE_CASE_  :			Union[str, Any]       ,							SCREAMING_SNAKE_CASE_  :			Optional[Any]       ,							SCREAMING_SNAKE_CASE_  :			Optional[int]       ,							SCREAMING_SNAKE_CASE_  :			Optional[Any]       ,							SCREAMING_SNAKE_CASE_  :			List[str]       )       ->     int:
					lowercase_       =					LlamaModel(config=a__       )
					model.to(a__       )
					model.eval()
					lowercase_       =					model(a__       ,							attention_mask=a__       )
					lowercase_       =					model(a__       )
					self.parent.assertEqual(result.last_hidden_state.shape       ,							(self.batch_size, self.seq_length, self.hidden_size)       )
	def 						_lowercase  (       self  :			Optional[Any]       ,							SCREAMING_SNAKE_CASE_  :			str       ,							SCREAMING_SNAKE_CASE_  :			List[Any]       ,							SCREAMING_SNAKE_CASE_  :			Dict       ,							SCREAMING_SNAKE_CASE_  :			Any       ,							SCREAMING_SNAKE_CASE_  :			Tuple       ,							SCREAMING_SNAKE_CASE_  :			Optional[int]       ,							SCREAMING_SNAKE_CASE_  :			List[str]       ,							SCREAMING_SNAKE_CASE_  :			Optional[Any]       ,							SCREAMING_SNAKE_CASE_  :			Dict       ,							)       ->     Union[str, Any]:
					lowercase_       =					True
					lowercase_       =					LlamaModel(a__       )
					model.to(a__       )
					model.eval()
					lowercase_       =					model(
					    a__       ,							attention_mask=a__       ,							encoder_hidden_states=a__       ,							encoder_attention_mask=a__       ,							)
					lowercase_       =					model(
					    a__       ,							attention_mask=a__       ,							encoder_hidden_states=a__       ,							)
					lowercase_       =					model(a__       ,							attention_mask=a__       )
					self.parent.assertEqual(result.last_hidden_state.shape       ,							(self.batch_size, self.seq_length, self.hidden_size)       )
	def 						_lowercase  (       self  :			Any       ,							SCREAMING_SNAKE_CASE_  :			Union[str, Any]       ,							SCREAMING_SNAKE_CASE_  :			Tuple       ,							SCREAMING_SNAKE_CASE_  :			Tuple       ,							SCREAMING_SNAKE_CASE_  :			int       ,							SCREAMING_SNAKE_CASE_  :			int       ,							SCREAMING_SNAKE_CASE_  :			str       ,							SCREAMING_SNAKE_CASE_  :			int       ,							SCREAMING_SNAKE_CASE_  :			List[Any]       ,							SCREAMING_SNAKE_CASE_  :			List[Any]       ,							)       ->     List[str]:
					lowercase_       =					LlamaForCausalLM(config=a__       )
					model.to(a__       )
					model.eval()
					lowercase_       =					model(a__       ,							attention_mask=a__       ,							labels=a__       )
					self.parent.assertEqual(result.logits.shape       ,							(self.batch_size, self.seq_length, self.vocab_size)       )
	def 						_lowercase  (       self  :			int       ,							SCREAMING_SNAKE_CASE_  :			str       ,							SCREAMING_SNAKE_CASE_  :			Tuple       ,							SCREAMING_SNAKE_CASE_  :			Union[str, Any]       ,							SCREAMING_SNAKE_CASE_  :			Optional[int]       ,							SCREAMING_SNAKE_CASE_  :			Any       ,							SCREAMING_SNAKE_CASE_  :			str       ,							SCREAMING_SNAKE_CASE_  :			Optional[int]       ,							SCREAMING_SNAKE_CASE_  :			Union[str, Any]       ,							SCREAMING_SNAKE_CASE_  :			int       ,							)       ->     Tuple:
					lowercase_       =					True
					lowercase_       =					True
					lowercase_       =					LlamaForCausalLM(config=a__       )
					model.to(a__       )
					model.eval()

					# first forward pass
					lowercase_       =					model(
					    a__       ,							attention_mask=a__       ,							encoder_hidden_states=a__       ,							encoder_attention_mask=a__       ,							use_cache=a__       ,							)
					lowercase_       =					outputs.past_key_values

					# create hypothetical multiple next token and extent to next_input_ids
					lowercase_       =					ids_tensor((self.batch_size, 3)       ,							config.vocab_size       )
					lowercase_       =					ids_tensor((self.batch_size, 3)       ,							vocab_size=2       )

					# append to next input_ids and
					lowercase_       =					torch.cat([input_ids, next_tokens]       ,							dim=-1       )
					lowercase_       =					torch.cat([input_mask, next_mask]       ,							dim=-1       )

					lowercase_       =					model(
					    a__       ,							attention_mask=a__       ,							encoder_hidden_states=a__       ,							encoder_attention_mask=a__       ,							output_hidden_states=a__       ,							)['''hidden_states'''][0]
					lowercase_       =					model(
					    a__       ,							attention_mask=a__       ,							encoder_hidden_states=a__       ,							encoder_attention_mask=a__       ,							past_key_values=a__       ,							output_hidden_states=a__       ,							)['''hidden_states'''][0]

					# select random slice
					lowercase_       =					ids_tensor((1,)       ,							output_from_past.shape[-1]       ).item()
					lowercase_       =					output_from_no_past[:, -3:, random_slice_idx].detach()
					lowercase_       =					output_from_past[:, :, random_slice_idx].detach()

					self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]       )

					# test that outputs are equal for slice
					self.parent.assertTrue(torch.allclose(a__       ,							a__       ,							atol=1e-3       )       )




	def 						_lowercase  (       self  :			Dict       )       ->     List[str]:
					lowercase_       =					self.prepare_config_and_inputs()
					(
					    (
					    lowercase_	
					)  ,	(
					    lowercase_	
					)  ,	(
					    lowercase_	
					)  ,	(
					    lowercase_	
					)  ,	(
					    lowercase_	
					)  ,	(
					    lowercase_	
					)  ,	(
					    lowercase_	
					)  ,		
					)       =					config_and_inputs
					lowercase_       =					{'''input_ids''': input_ids, '''attention_mask''': input_mask}
					return config, inputs_dict

@require_torch
class  lowercase__(				_UpperCAmelCase ,						_UpperCAmelCase ,						_UpperCAmelCase ,						unittest.TestCase							):



	"""simple docstring"""




	a			:List[Any]         =	(LlamaModel, LlamaForCausalLM, LlamaForSequenceClassification) if is_torch_available() else ()
	a			:List[str]         =	(LlamaForCausalLM,) if is_torch_available() else ()
	a			:Dict         =	(
	    {
	        'feature-extraction': LlamaModel,
	        'text-classification': LlamaForSequenceClassification,
	        'text-generation': LlamaForCausalLM,
	        'zero-shot': LlamaForSequenceClassification,
	    }
	    if is_torch_available()
	    else {}
	)
	a			:Any         =	False
	a			:Union[str, Any]         =	False
	def 						_lowercase  (       self  :			Optional[Any]       )       ->     Dict:
					lowercase_       =					LlamaModelTester(self       )
					lowercase_       =					ConfigTester(self       ,							config_class=a__       ,							hidden_size=3_7       )
	def 						_lowercase  (       self  :			Optional[Any]       )       ->     Dict:
					self.config_tester.run_common_tests()
	def 						_lowercase  (       self  :			List[str]       )       ->     Union[str, Any]:
					lowercase_       =					self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_model(*a__       )
	def 						_lowercase  (       self  :			List[Any]       )       ->     Optional[Any]:
					lowercase_       =					self.model_tester.prepare_config_and_inputs()
					for type in ["absolute", "relative_key", "relative_key_query"]:
									lowercase_       =					type
									self.model_tester.create_and_check_model(*a__       )
	def 						_lowercase  (       self  :			Optional[int]       )       ->     Optional[Any]:
					lowercase_  ,	lowercase_       =					self.model_tester.prepare_config_and_inputs_for_common()
					lowercase_       =					3
					lowercase_       =					input_dict['''input_ids''']
					lowercase_       =					input_ids.ne(1       ).to(a__       )
					lowercase_       =					ids_tensor([self.model_tester.batch_size]       ,							self.model_tester.type_sequence_label_size       )
					lowercase_       =					LlamaForSequenceClassification(a__       )
					model.to(a__       )
					model.eval()
					lowercase_       =					model(a__       ,							attention_mask=a__       ,							labels=a__       )
					self.assertEqual(result.logits.shape       ,							(self.model_tester.batch_size, self.model_tester.num_labels)       )
	def 						_lowercase  (       self  :			Union[str, Any]       )       ->     Dict:
					lowercase_  ,	lowercase_       =					self.model_tester.prepare_config_and_inputs_for_common()
					lowercase_       =					3
					lowercase_       =					'''single_label_classification'''
					lowercase_       =					input_dict['''input_ids''']
					lowercase_       =					input_ids.ne(1       ).to(a__       )
					lowercase_       =					ids_tensor([self.model_tester.batch_size]       ,							self.model_tester.type_sequence_label_size       )
					lowercase_       =					LlamaForSequenceClassification(a__       )
					model.to(a__       )
					model.eval()
					lowercase_       =					model(a__       ,							attention_mask=a__       ,							labels=a__       )
					self.assertEqual(result.logits.shape       ,							(self.model_tester.batch_size, self.model_tester.num_labels)       )
	def 						_lowercase  (       self  :			List[Any]       )       ->     Optional[int]:
					lowercase_  ,	lowercase_       =					self.model_tester.prepare_config_and_inputs_for_common()
					lowercase_       =					3
					lowercase_       =					'''multi_label_classification'''
					lowercase_       =					input_dict['''input_ids''']
					lowercase_       =					input_ids.ne(1       ).to(a__       )
					lowercase_       =					ids_tensor(
					    [self.model_tester.batch_size, config.num_labels]       ,							self.model_tester.type_sequence_label_size       ).to(torch.float       )
					lowercase_       =					LlamaForSequenceClassification(a__       )
					model.to(a__       )
					model.eval()
					lowercase_       =					model(a__       ,							attention_mask=a__       ,							labels=a__       )
					self.assertEqual(result.logits.shape       ,							(self.model_tester.batch_size, self.model_tester.num_labels)       )
	@unittest.skip('''LLaMA buffers include complex numbers, which breaks this test'''       )
	def 						_lowercase  (       self  :			Union[str, Any]       )       ->     Any:
					pass




	@parameterized.expand([('''linear''',), ('''dynamic''',)]       )
	def 						_lowercase  (       self  :			Optional[Any]       ,							SCREAMING_SNAKE_CASE_  :			Optional[int]       )       ->     int:
					lowercase_  ,	lowercase_       =					self.model_tester.prepare_config_and_inputs_for_common()
					lowercase_       =					ids_tensor([1, 1_0]       ,							config.vocab_size       )
					lowercase_       =					ids_tensor([1, int(config.max_position_embeddings * 1.5       )]       ,							config.vocab_size       )

					set_seed(4_2       )  # Fixed seed at init time so the two models get the same random weights
					lowercase_       =					LlamaModel(a__       )
					original_model.to(a__       )
					original_model.eval()
					lowercase_       =					original_model(a__       ).last_hidden_state
					lowercase_       =					original_model(a__       ).last_hidden_state

					set_seed(4_2       )  # Fixed seed at init time so the two models get the same random weights
					lowercase_       =					{'''type''': scaling_type, '''factor''': 1_0.0}
					lowercase_       =					LlamaModel(a__       )
					scaled_model.to(a__       )
					scaled_model.eval()
					lowercase_       =					scaled_model(a__       ).last_hidden_state
					lowercase_       =					scaled_model(a__       ).last_hidden_state

					# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
					# maximum sequence length, so the outputs for the short input should match.
					if scaling_type == "dynamic":
									self.assertTrue(torch.allclose(a__       ,							a__       ,							atol=1e-5       )       )
					else:
									self.assertFalse(torch.allclose(a__       ,							a__       ,							atol=1e-5       )       )

					# The output should be different for long inputs
					self.assertFalse(torch.allclose(a__       ,							a__       ,							atol=1e-5       )       )

@require_torch
class  lowercase__(				unittest.TestCase							):



	"""simple docstring"""




	@unittest.skip('''Logits are not exactly the same, once we fix the instabalities somehow, will update!'''       )
	@slow
	def 						_lowercase  (       self  :			Tuple       )       ->     str:
					lowercase_       =					[1, 3_0_6, 4_6_5_8, 2_7_8, 6_5_9_3, 3_1_0, 2_8_3_4, 3_3_8]
					lowercase_       =					LlamaForCausalLM.from_pretrained('''meta-llama/Llama-2-7b-hf'''       ,							device_map='''auto'''       )
					lowercase_       =					model(torch.tensor([input_ids]       )       )
					# Expected mean on dim = -1
					lowercase_       =					torch.tensor([[-6.65_50, -4.12_27, -4.98_59, -3.24_06, 0.82_62, -3.00_33, 1.29_64, -3.36_99]]       )
					torch.testing.assert_close(out.mean(-1       )       ,							a__       ,							atol=1e-2       ,							rtol=1e-2       )
					# slicing logits[0, 0, 0:30]
					# fmt: off
					lowercase_       =					torch.tensor([-1_2.8_2_8_1, -7.44_53, -0.46_39, -8.06_25, -7.25_00, -8.00_00, -6.48_83, -7.76_95, -7.84_38, -7.03_12, -6.21_88, -7.13_28, -1.84_96, 1.99_61, -8.62_50, -6.72_27, -1_2.8_2_8_1, -6.94_92, -7.07_42, -7.78_52, -7.58_20, -7.90_62, -6.93_75, -7.98_05, -8.34_38, -8.15_62, -8.04_69, -7.62_50, -7.74_22, -7.33_98,]       )
					# fmt: on
					torch.testing.assert_close(out[0, 0, :3_0]       ,							a__       ,							atol=1e-5       ,							rtol=1e-5       )
	@unittest.skip('''Logits are not exactly the same, once we fix the instabalities somehow, will update!'''       )
	@slow
	def 						_lowercase  (       self  :			str       )       ->     List[Any]:
					lowercase_       =					[1, 3_0_6, 4_6_5_8, 2_7_8, 6_5_9_3, 3_1_0, 2_8_3_4, 3_3_8]
					lowercase_       =					LlamaForCausalLM.from_pretrained('''meta-llama/Llama-2-13b-hf'''       ,							device_map='''auto'''       )
					lowercase_       =					model(torch.tensor(a__       )       )
					# Expected mean on dim = -1
					lowercase_       =					torch.tensor([[-2.06_22, -1.27_94, -1.16_38, -0.97_88, -1.46_03, -1.02_38, -1.78_93, -1.44_11]]       )
					torch.testing.assert_close(out.mean(-1       )       ,							a__       ,							atol=1e-2       ,							rtol=1e-2       )
					# slicing logits[0, 0, 0:30]
					# fmt: off
					lowercase_       =					torch.tensor([-8.14_06, -8.05_47, 2.74_61, -1.23_44, -0.14_48, -1.82_62, -1.00_20, -1.81_54, -1.68_95, -1.85_16, -2.35_74, -0.92_77, 3.75_98, 6.57_42, -1.29_98, -0.11_77, -8.14_06, -2.96_88, -2.91_99, -3.16_99, -3.52_54, -2.35_55, -2.79_88, -3.41_41, -2.82_62, -4.51_95, -3.33_79, -3.31_64, -2.78_32, -3.02_73]       )
					# fmt: on
					torch.testing.assert_close(out[0, 0, :3_0]       ,							a__       ,							atol=1e-5       ,							rtol=1e-5       )
	@unittest.skip('''Logits are not exactly the same, once we fix the instabalities somehow, will update!'''       )
	@slow
	def 						_lowercase  (       self  :			Any       )       ->     Optional[int]:
					lowercase_       =					[1, 3_0_6, 4_6_5_8, 2_7_8, 6_5_9_3, 3_1_0, 2_8_3_4, 3_3_8]
					lowercase_       =					LlamaForCausalLM.from_pretrained('''meta-llama/Llama-2-13b-chat-hf'''       ,							device_map='''auto'''       )
					lowercase_       =					model(torch.tensor(a__       )       )
					# Expected mean on dim = -1
					lowercase_       =					torch.tensor([[-0.85_62, -1.85_20, -0.75_51, -0.41_62, -1.51_61, -1.20_38, -2.48_23, -2.32_54]]       )
					torch.testing.assert_close(out.mean(-1       )       ,							a__       ,							atol=1e-2       ,							rtol=1e-2       )
					# slicing logits[0, 0, 0:30]
					# fmt: off
					lowercase_       =					torch.tensor([-2.22_27, 4.88_28, 0.90_23, -0.45_78, -0.78_71, -0.10_33, -0.62_21, -0.57_86, -0.78_03, -1.06_74, -1.29_20, -0.15_70, 0.80_08, 2.07_23, -0.94_97, 0.27_71, -2.22_27, -0.76_12, -1.43_46, -1.20_61, -1.64_26, -0.30_00, -0.71_39, -1.19_34, -1.86_91, -1.69_73, -1.59_47, -1.27_05, -0.35_23, -0.55_13]       )
					# fmt: on
					torch.testing.assert_close(out.mean(-1       )       ,							a__       ,							atol=1e-2       ,							rtol=1e-2       )
	@unittest.skip(
	    '''Logits are not exactly the same, once we fix the instabalities somehow, will update! Also it is gonna be a `too_slow` test'''       )
	@slow
	def 						_lowercase  (       self  :			Optional[int]       )       ->     Dict:
					lowercase_       =					[1, 3_0_6, 4_6_5_8, 2_7_8, 6_5_9_3, 3_1_0, 2_8_3_4, 3_3_8]
					lowercase_       =					LlamaForCausalLM.from_pretrained('''meta-llama/Llama-2-70b-hf'''       ,							device_map='''auto'''       )
					lowercase_       =					model(torch.tensor(a__       )       )

					lowercase_       =					torch.tensor(
					    [[-4.23_27, -3.33_60, -4.66_65, -4.76_31, -1.81_80, -3.41_70, -1.42_11, -3.18_10]]       ,							dtype=torch.floataa       )
					torch.testing.assert_close(out.mean(-1       )       ,							a__       ,							atol=1e-2       ,							rtol=1e-2       )
					# fmt: off
					lowercase_       =					torch.tensor([-9.49_22, -3.95_51, 1.79_98, -5.67_58, -5.10_55, -5.89_84, -4.83_20, -6.80_86, -6.53_91, -5.61_72, -5.58_20, -5.53_52, 1.78_81, 3.62_89, -6.51_17, -3.47_85, -9.50_00, -6.03_52, -6.81_25, -6.01_95, -6.68_36, -5.47_27, -6.28_12, -6.03_91, -7.33_98, -7.42_97, -7.48_44, -6.58_20, -5.87_89, -5.53_12]       )
					# fmt: on
					torch.testing.assert_close(out[0, 0, :3_0]       ,							a__       ,							atol=1e-5       ,							rtol=1e-5       )




	@unittest.skip('''Model is curently gated'''       )
	@slow
	def 						_lowercase  (       self  :			List[str]       )       ->     int:
					lowercase_       =					'''Simply put, the theory of relativity states that 1) the laws of physics are the same everywhere in the universe and 2) the passage of time and the length of objects can vary depending on the observer\'s frame of reference.\n\nThe first part of the theory, that the laws of physics are the same everywhere, is known as the "princi'''
					lowercase_       =					'''Simply put, the theory of relativity states that '''
					lowercase_       =					LlamaTokenizer.from_pretrained('''meta-llama/Llama-2-13b-chat-hf'''       )
					lowercase_       =					tokenizer.encode(a__       ,							return_tensors='''pt'''       )
					lowercase_       =					LlamaForCausalLM.from_pretrained(
					    '''meta-llama/Llama-2-13b-chat-hf'''       ,							device_map='''sequential'''       ,							use_safetensors=a__       )

					# greedy generation outputs
					lowercase_       =					model.generate(a__       ,							max_new_tokens=6_4       ,							top_p=a__       ,							temperature=1       ,							do_sample=a__       )
					lowercase_       =					tokenizer.decode(generated_ids[0]       ,							skip_special_tokens=a__       )
					self.assertEqual(a__       ,							a__       )











import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


snake_case_				=			logging.getLogger(__name__)

@dataclass(frozen=_UpperCAmelCase  )
class 		SCREAMING_SNAKE_CASE__				:
      A_						:				str
      A_						:				str
      A_						:				Optional[str]							=    None
      A_						:				Optional[str]							=    None
      A_						:				Optional[str]							=    None






@dataclass(frozen=_UpperCAmelCase  )
class 		SCREAMING_SNAKE_CASE__				:
      A_						:				List[int]
      A_						:				Optional[List[int]]							=    None
      A_						:				Optional[List[int]]							=    None
      A_						:				Optional[Union[int, float]]							=    None
      A_						:				Optional[int]							=    None


if is_torch_available():
    import torch
    from torch.utils.data import Dataset

    class 		SCREAMING_SNAKE_CASE__				(       _UpperCAmelCase  ):
          A_						:				List[InputFeatures]


          def __init__(self       :      int				,			a__       :      str				,			a__       :      PreTrainedTokenizer				,			a__       :      str				,			a__       :      Optional[int] = None				,			a__       :      List[Any]=False				,			a__       :      bool = False				,			):





             """simple docstring"""



             __snake_case							       =   hans_processors[task]()

             __snake_case							       =   os.path.join(
                 a__				,			'''cached_{}_{}_{}_{}'''.format(
                     '''dev''' if evaluate else '''train'''				,			tokenizer.__class__.__name__				,			str(a__      )				,			a__				,			)				,			)
             __snake_case							       =   processor.get_labels()
             if tokenizer.__class__ in (
                 RobertaTokenizer,
                 RobertaTokenizerFast,
                 XLMRobertaTokenizer,
                 BartTokenizer,
                 BartTokenizerFast,
             ):
                # HACK(label indices are swapped in RoBERTa pretrained model)
                __snake_case       ,	__snake_case							       =   label_list[2], label_list[1]
             __snake_case							       =   label_list

             # Make sure only the first process in distributed training processes the dataset,
             # and the others will use the cache.
             __snake_case							       =   cached_features_file + '''.lock'''
             with FileLock(a__      ):
                if os.path.exists(a__      ) and not overwrite_cache:
                   logger.info(f"""Loading features from cached file {cached_features_file}"""      )
                   __snake_case							       =   torch.load(a__      )
                else:
                   logger.info(f"""Creating features from dataset file at {data_dir}"""      )

                   __snake_case							       =   (
                       processor.get_dev_examples(a__      ) if evaluate else processor.get_train_examples(a__      )
                   )

                   logger.info('''Training examples: %s'''				,			len(a__      )      )
                   __snake_case							       =   hans_convert_examples_to_features(a__				,			a__				,			a__				,			a__      )
                   logger.info('''Saving features into cached file %s'''				,			a__      )
                   torch.save(self.features				,			a__      )


          def __len__(self       :      int      ):





             """simple docstring"""



             return len(self.features      )


          def __getitem__(self       :      Dict				,			a__       :      List[Any]      ):





             """simple docstring"""



             return self.features[i]


          def    a (self       :      List[Any]      ):





             """simple docstring"""



             return self.label_list


if is_tf_available():
    import tensorflow as tf

    class 		SCREAMING_SNAKE_CASE__				:
          A_						:				List[InputFeatures]


          def __init__(self       :      Tuple				,			a__       :      str				,			a__       :      PreTrainedTokenizer				,			a__       :      str				,			a__       :      Optional[int] = 128				,			a__       :      Any=False				,			a__       :      bool = False				,			):





             """simple docstring"""



             __snake_case							       =   hans_processors[task]()
             __snake_case							       =   processor.get_labels()
             if tokenizer.__class__ in (
                 RobertaTokenizer,
                 RobertaTokenizerFast,
                 XLMRobertaTokenizer,
                 BartTokenizer,
                 BartTokenizerFast,
             ):
                # HACK(label indices are swapped in RoBERTa pretrained model)
                __snake_case       ,	__snake_case							       =   label_list[2], label_list[1]
             __snake_case							       =   label_list

             __snake_case							       =   processor.get_dev_examples(a__      ) if evaluate else processor.get_train_examples(a__      )
             __snake_case							       =   hans_convert_examples_to_features(a__				,			a__				,			a__				,			a__      )

             def gen():
                for ex_index, ex in tqdm.tqdm(enumerate(self.features      )				,			desc='''convert examples to features'''      ):
                   if ex_index % 1_0000 == 0:
                      logger.info('''Writing example %d of %d''' % (ex_index, len(a__      ))      )

                   yield (
                       {
                           "example_id": 0,
                           "input_ids": ex.input_ids,
                           "attention_mask": ex.attention_mask,
                           "token_type_ids": ex.token_type_ids,
                       },
                       ex.label,
                   )

             __snake_case							       =   tf.data.Dataset.from_generator(
                 a__				,			(
                     {
                         '''example_id''': tf.intaa,
                         '''input_ids''': tf.intaa,
                         '''attention_mask''': tf.intaa,
                         '''token_type_ids''': tf.intaa,
                     },
                     tf.intaa,
                 )				,			(
                     {
                         '''example_id''': tf.TensorShape([]      ),
                         '''input_ids''': tf.TensorShape([None, None]      ),
                         '''attention_mask''': tf.TensorShape([None, None]      ),
                         '''token_type_ids''': tf.TensorShape([None, None]      ),
                     },
                     tf.TensorShape([]      ),
                 )				,			)


          def    a (self       :      Union[str, Any]      ):





             """simple docstring"""



             return self.dataset


          def __len__(self       :      Dict      ):





             """simple docstring"""



             return len(self.features      )


          def __getitem__(self       :      Any				,			a__       :      Dict      ):





             """simple docstring"""



             return self.features[i]


          def    a (self       :      str      ):





             """simple docstring"""



             return self.label_list

class 		SCREAMING_SNAKE_CASE__				(       _UpperCAmelCase  ):


      def    a (self       :      Dict				,			a__       :      Dict      ):





         """simple docstring"""



         return self._create_examples(self._read_tsv(os.path.join(a__				,			'''heuristics_train_set.txt'''      )      )				,			'''train'''      )


      def    a (self       :      Optional[int]				,			a__       :      Tuple      ):





         """simple docstring"""



         return self._create_examples(self._read_tsv(os.path.join(a__				,			'''heuristics_evaluation_set.txt'''      )      )				,			'''dev'''      )


      def    a (self       :      int      ):





         """simple docstring"""



         return ["contradiction", "entailment", "neutral"]


      def    a (self       :      Any				,			a__       :      Optional[int]				,			a__       :      List[Any]      ):





         """simple docstring"""



         __snake_case							       =   []
         for i, line in enumerate(a__      ):
            if i == 0:
               continue
            __snake_case							       =   '''%s-%s''' % (set_type, line[0])
            __snake_case							       =   line[5]
            __snake_case							       =   line[6]
            __snake_case							       =   line[7][2:] if line[7].startswith('''ex'''      ) else line[7]
            __snake_case							       =   line[0]
            examples.append(InputExample(guid=a__				,			text_a=a__				,			text_b=a__				,			label=a__				,			pairID=a__      )      )
         return examples






def lowerCamelCase__    (							snake_case_	:  List[InputExample]       ,      snake_case_	:  List[str]       ,      snake_case_	:  int       ,      snake_case_	:  PreTrainedTokenizer       ,      )      ->				List[str]:
   __snake_case							       =   {label: i for i, label in enumerate(snake_case_       )}

   __snake_case							       =   []
   for ex_index, example in tqdm.tqdm(enumerate(snake_case_       )       ,      desc='''convert examples to features'''       ):
      if ex_index % 1_0000 == 0:
         logger.info('''Writing example %d''' % (ex_index)       )

      __snake_case							       =   tokenizer(
          example.text_a       ,      example.text_b       ,      add_special_tokens=snake_case_       ,      max_length=snake_case_       ,      padding='''max_length'''       ,      truncation=snake_case_       ,      return_overflowing_tokens=snake_case_       ,      )

      __snake_case							       =   label_map[example.label] if example.label in label_map else 0

      __snake_case							       =   int(example.pairID       )

      features.append(InputFeatures(**snake_case_       ,      label=snake_case_       ,      pairID=snake_case_       )       )

   for i, example in enumerate(examples[:5]       ):
      logger.info('''*** Example ***'''       )
      logger.info(f"""guid: {example}"""       )
      logger.info(f"""features: {features[i]}"""       )

   return features


snake_case_				=			{
    'hans': 3,
}







snake_case_				=			{
    'hans': HansProcessor,
}

