




"""simple docstring"""

import copy
import logging
import random

import joblib
import numpy as np
import torch
import torch.nn as nn
from torch.utils.data import DataLoader
from tqdm import tqdm

from transformers import AdamW, GPTaLMHeadModel, get_linear_schedule_with_warmup


UpperCamelCase__			:     List[str]			       =				logging.getLogger(__name__)




def    snake_case__						(							lowerCAmelCase :		int		):
					random.seed(lowerCAmelCase		)
					np.random.seed(lowerCAmelCase		)
					torch.manual_seed(lowerCAmelCase		)
					torch.cuda.manual_seed_all(lowerCAmelCase		)




def    snake_case__						(							lowerCAmelCase :		Dict					,       lowerCAmelCase :		Tuple					,       lowerCAmelCase :		Any		):
					model.eval()
					__snake_case        =						next(model.parameters()		).device
					__snake_case        =						1
					__snake_case        =						torch.zeros((eval_batch_size, context_len)					,       dtype=torch.long					,       device=lowerCAmelCase		)
					__snake_case        =						DataLoader(lowerCAmelCase					,       shuffle=lowerCAmelCase					,       batch_size=lowerCAmelCase		)
					__snake_case        =						torch.zeros(1					,       device=lowerCAmelCase		)
					__snake_case        =						0
					for batch in eval_dataloader:
										batch.to(lowerCAmelCase		)
										# pad
										context.zero_()
										for i in range(lowerCAmelCase		):
															__snake_case        =						batch[i]
										__snake_case        =						model(lowerCAmelCase					,       labels=lowerCAmelCase		)
										eval_loss += outputs[0].sum().item()
										nb_eval_examples += batch.size(0		)
					__snake_case        =						eval_loss / nb_eval_examples
					__snake_case        =						torch.exp(lowerCAmelCase		)
					model.train()
					return perplexity




def    snake_case__						(							lowerCAmelCase :		Tuple="gpt2"		):
					__snake_case        =						GPTaLMHeadModel.from_pretrained(lowerCAmelCase					,       output_hidden_states=lowerCAmelCase		)
					torch.save(model.state_dict()					,       model_name + """local.pt"""		)
					return model




def    snake_case__						(							lowerCAmelCase :		int					,       lowerCAmelCase :		Optional[int]					,       lowerCAmelCase :		List[str]		):
					__snake_case        =						copy.deepcopy(lowerCAmelCase		)
					model.to(lowerCAmelCase		)

					__snake_case        =						["""bias""", """LayerNorm.weight"""]
					__snake_case        =						[
					    {
					        """params""": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay		)],
					        """weight_decay""": 0.0,
					    },
					    {"""params""": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay		)], """weight_decay""": 0.0},
					]
					__snake_case        =						AdamW(lowerCAmelCase					,       lr=5e-5					,       eps=1e-8		)
					__snake_case        =						get_linear_schedule_with_warmup(lowerCAmelCase					,       0					,       lowerCAmelCase		)
					torch.cuda.empty_cache()
					return model, lm_optimizer, lm_scheduler




def    snake_case__						(							lowerCAmelCase :		List[str]					,       lowerCAmelCase :		List[str]					,       lowerCAmelCase :		Optional[int]					,       lowerCAmelCase :		List[Any]		):
					__snake_case        =						np.array(lowerCAmelCase		).mean()
					__snake_case        =						np.array(lowerCAmelCase		).std()
					__snake_case        =						(real_perps - avg) / std
					__snake_case        =						list(zip(lowerCAmelCase					,       lowerCAmelCase					,       lowerCAmelCase		)		)
					joblib.dump(lowerCAmelCase					,       lowerCAmelCase		)




def    snake_case__						(							lowerCAmelCase :		Union[str, Any]					,       lowerCAmelCase :		Optional[Any]					,       lowerCAmelCase :		Optional[Any]					,       lowerCAmelCase :		int					,       lowerCAmelCase :		List[str]					,       lowerCAmelCase :		str					,       lowerCAmelCase :		Dict					,       lowerCAmelCase :		Optional[Any]="dev.jbl"					,       lowerCAmelCase :		List[Any]=recopy_gpta					,       ):
					__snake_case        =						[]
					__snake_case        =						[]
					__snake_case        =						[]

					# Initialize the transformer model
					__snake_case        =						copy.deepcopy(lowerCAmelCase		)
					orig_model.to(device="""cpu"""		)
					torch.cuda.empty_cache()

					# Compute perplexity of initial transformer model for comparison
					model.train()
					__snake_case   ,   __snake_case   ,   __snake_case        =						recopy_model(lowerCAmelCase					,       lowerCAmelCase					,       lowerCAmelCase		)

					for step in tqdm(range(lowerCAmelCase		)		):
										__snake_case        =						torch.zeros((1, context_len)					,       dtype=torch.long					,       device=lowerCAmelCase		)
										__snake_case        =						random.choice(lowerCAmelCase		)
										__snake_case        =						random.randint(0					,       len(story[0]		) - context_len - 1		)
										__snake_case        =						story[0][start : start + context_len]
										lm_optimizer.zero_grad()
										__snake_case        =						model(lowerCAmelCase					,       labels=lowerCAmelCase		)
										__snake_case        =						outputs[0]
										__snake_case        =						compute_perplexity(lowerCAmelCase					,       lowerCAmelCase					,       lowerCAmelCase		)
										model.train()
										lm_loss.backward()
										# Do LM backprop
										torch.nn.utils.clip_grad_norm_(model.parameters()					,       3.0		)
										lm_optimizer.step()
										lm_scheduler.step()  # Update learning rate schedule

										# Compute perplexity after back-propagating on the selected context
										__snake_case        =						compute_perplexity(lowerCAmelCase					,       lowerCAmelCase					,       lowerCAmelCase		)

										# Periodically save the stored (X, IG(X)) pairs
										if step % 1_000 == 0 and step > 1:
															intermittent_save(lowerCAmelCase					,       lowerCAmelCase					,       lowerCAmelCase					,       lowerCAmelCase		)

										# Reset the pretrained model to the original pretrained GPT-2 weights after each iteration
										__snake_case   ,   __snake_case   ,   __snake_case        =						recopy_model(lowerCAmelCase					,       lowerCAmelCase					,       lowerCAmelCase		)

										past_perps.append(past_perp.item()		)
										real_perps.append(orig_perp - real_perp.item()		)
										contexts.append(np.array(context.cpu()		)		)

					intermittent_save(lowerCAmelCase					,       lowerCAmelCase					,       lowerCAmelCase					,       lowerCAmelCase		)




def    snake_case__						(							lowerCAmelCase :		List[str]					,       lowerCAmelCase :		int="data/tokenized_stories_train_wikitext103.jbl"					,       lowerCAmelCase :		Tuple=100					,       lowerCAmelCase :		Union[str, Any]=1_026					,       lowerCAmelCase :		Tuple=True		):
					__snake_case        =						joblib.load(lowerCAmelCase		)
					print("""data loaded"""		)
					__snake_case        =						[]
					if trim:
										for i, example in enumerate(lowerCAmelCase		):
															if len(example[0]		) > min_len:
																				__snake_case        =						random.randint(0					,       len(example[0]		) - context_len - 1		)
																				objective_set.append(example[0, start : start + context_len]		)
															if len(lowerCAmelCase		) >= number:
																				break
										__snake_case        =						[]
										for j in range(i + 1					,       len(lowerCAmelCase		)		):
															if len(data[j][0]		) > min_len:
																				train_data.append(data[j]		)
					else:
										__snake_case        =						data[0:number]
										__snake_case        =						data[number:]

					joblib.dump(lowerCAmelCase					,       """objective_set.jbl"""		)
					print("""objective set saved"""		)
					return train_data, objective_set




def    snake_case__						(							lowerCAmelCase :		Optional[int]					,       lowerCAmelCase :		Tuple					,       lowerCAmelCase :		str					,       lowerCAmelCase :		Any					,       lowerCAmelCase :		int=50					,       lowerCAmelCase :		Optional[int]="secondary_learner.pt"		):
					__snake_case        =						torch.device("""cuda:0""" if torch.cuda.is_available() else """cpu"""		)
					# We will use the first 512 pairs from our dataset as a test set for
					# our secondary learner and the rest to train
					__snake_case        =						train_dataset[:512]
					__snake_case        =						train_dataset[512:]
					__snake_case        =						DataLoader(lowerCAmelCase					,       shuffle=lowerCAmelCase					,       batch_size=lowerCAmelCase		)
					__snake_case        =						DataLoader(lowerCAmelCase					,       shuffle=lowerCAmelCase					,       batch_size=lowerCAmelCase		)

					# secondary learner model set up
					__snake_case        =						nn.MSELoss()
					__snake_case        =						nn.MSELoss(reduction="""sum"""		)
					secondary_learner.to(lowerCAmelCase		)
					__snake_case        =						torch.optim.Adam(secondary_learner.parameters()					,       lr=0.0_0_0_0_1		)
					secondary_learner.train()

					# TODO in original code this is written as number of actual batches seen
					# not number of items seen but other places it is number of items instead.
					# improve consistency! changed this to epochs for clarity
					__snake_case        =						float("""inf"""		)
					# Iterate through batches until we've used max_steps batches
					for epoch in range(int(lowerCAmelCase		)		):
										__snake_case        =						0.0
										secondary_learner.train()
										for step, batch in enumerate(lowerCAmelCase		):
															__snake_case        =						batch[0].to(lowerCAmelCase		)
															__snake_case        =						batch[1].to(lowerCAmelCase		)
															__snake_case        =						secondary_learner(lowerCAmelCase		)
															q_optimizer.zero_grad()
															__snake_case        =						loss(lowerCAmelCase					,       real_q.float()		)
															q_loss.backward()
															q_optimizer.step()
															tr_q_loss += q_loss.item()

															# model trains fairly quickly so we won't wait for a full epoch
															# eval is triggered at eval_freq and end of epochs
															if (step % eval_freq == 0 and step > 0) or ((step + 1) == len(lowerCAmelCase		)):
																				__snake_case        =						tr_q_loss / (step + 1)

																				secondary_learner.eval()
																				__snake_case        =						0.0
																				__snake_case        =						0.0
																				__snake_case        =						[]
																				__snake_case        =						[]
																				# Compute performance of the secondary learner after this batch
																				for stepa, batcha in enumerate(lowerCAmelCase		):
																									__snake_case        =						batcha[0].to(lowerCAmelCase		)
																									__snake_case        =						batcha[1].to(lowerCAmelCase		)
																									__snake_case        =						secondary_learner(lowerCAmelCase		)
																									q_lossa += test_loss(lowerCAmelCase					,       lowerCAmelCase		).item()
																									sum_qa += torch.sum(lowerCAmelCase		).item()
																									for ei, i in enumerate(predicted_qa.cpu().detach().numpy()		):
																														predicted.append(i.item()		)
																									for ei, i in enumerate(real_qa.cpu().detach().numpy()		):
																														actual.append(i.item()		)

																				q_lossa /= len(lowerCAmelCase		)
																				print(
																				    """Epoch: """					,       lowerCAmelCase					,       """step: """					,       lowerCAmelCase					,       """Avg. q:"""					,       sum_qa / len(lowerCAmelCase		)					,       """Train Loss: """					,       lowerCAmelCase					,       """Test Loss: """					,       lowerCAmelCase					,       )
																				if q_lossa < best_test_loss:
																									joblib.dump((predicted, actual)					,       """pred_vs_actual.jbl"""		)
																									torch.save(secondary_learner.state_dict()					,       lowerCAmelCase		)
																									__snake_case        =						q_lossa

															secondary_learner.train()
					return secondary_learner

class      A__   (   nn.Module  ):




	def __init__( self	:						List[Any]   ,     lowercase	:						Any						)       ->			int:
						super(lowercase   ,     self						).__init__()
						__snake_case        =						model.transformer.wte
						__snake_case        =						copy.deepcopy(model.transformer.wte.weight						)
						__snake_case        =						nn.Convad(self.embeddings.weight.size(1						)   ,     256   ,     3   ,     padding=1						)
						__snake_case        =						nn.Sequential(nn.Linear(256   ,     32						)   ,     nn.Dropout(p=0.1						)   ,     nn.Linear(32   ,     32						)   ,     nn.Linear(32   ,     1						)						)




	def 				a_		( self	:						Tuple   ,     lowercase	:						List[str]						)       ->			Tuple:
						__snake_case        =						torch.max(self.conv(self.embeddings(lowercase						).squeeze(1						).transpose(1   ,     2						)						)   ,     2						)[0]
						__snake_case        =						self.fc(lowercase						)
						return qs.squeeze(1						)




	@classmethod
	def 				a_		( cls	:						Tuple   ,     lowercase	:						int   ,     lowercase	:						List[str]						)       ->			str:
						__snake_case        =						cls(lowercase						)  # this calls __init__
						__snake_case        =						torch.load(lowercase						)
						secondary_learner.load_state_dict(lowercase						)
						__snake_case        =						model.transformer.wte
						__snake_case        =						copy.deepcopy(model.transformer.wte.weight						)
						return secondary_learner








"""simple docstring"""

import tempfile
import unittest

from transformers import DPRConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import DPRContextEncoder, DPRQuestionEncoder, DPRReader, DPRReaderTokenizer
		from transformers.models.dpr.modeling_dpr import (
		    DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
		    DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
		    DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
		)
class      A__   :




	def __init__( self	:						Optional[int]   ,     lowercase	:						List[Any]   ,     lowercase	:						Optional[int]=13   ,     lowercase	:						List[Any]=7   ,     lowercase	:						Any=False   ,     lowercase	:						Optional[int]=True   ,     lowercase	:						Any=True   ,     lowercase	:						Optional[Any]=True   ,     lowercase	:						Tuple=99   ,     lowercase	:						Union[str, Any]=32   ,     lowercase	:						Any=5   ,     lowercase	:						Dict=4   ,     lowercase	:						Dict=37   ,     lowercase	:						Tuple="gelu"   ,     lowercase	:						str=0.1   ,     lowercase	:						Union[str, Any]=0.1   ,     lowercase	:						Any=512   ,     lowercase	:						Optional[Any]=16   ,     lowercase	:						Any=2   ,     lowercase	:						List[str]=0.02   ,     lowercase	:						List[str]=3   ,     lowercase	:						Optional[int]=4   ,     lowercase	:						Any=None   ,     lowercase	:						Optional[int]=0   ,     )       ->			Optional[Any]:
						__snake_case        =						parent
						__snake_case        =						batch_size
						__snake_case        =						seq_length
						__snake_case        =						is_training
						__snake_case        =						use_input_mask
						__snake_case        =						use_token_type_ids
						__snake_case        =						use_labels
						__snake_case        =						vocab_size
						__snake_case        =						hidden_size
						__snake_case        =						num_hidden_layers
						__snake_case        =						num_attention_heads
						__snake_case        =						intermediate_size
						__snake_case        =						hidden_act
						__snake_case        =						hidden_dropout_prob
						__snake_case        =						attention_probs_dropout_prob
						__snake_case        =						max_position_embeddings
						__snake_case        =						type_vocab_size
						__snake_case        =						type_sequence_label_size
						__snake_case        =						initializer_range
						__snake_case        =						num_labels
						__snake_case        =						num_choices
						__snake_case        =						scope
						__snake_case        =						projection_dim




	def 				a_		( self	:						str						)       ->			Optional[Any]:
						__snake_case        =						ids_tensor([self.batch_size, self.seq_length]   ,     self.vocab_size						)

						__snake_case        =						None
						if self.use_input_mask:
											__snake_case        =						random_attention_mask([self.batch_size, self.seq_length]						)

						__snake_case        =						None
						if self.use_token_type_ids:
											__snake_case        =						ids_tensor([self.batch_size, self.seq_length]   ,     self.type_vocab_size						)

						__snake_case        =						None
						__snake_case        =						None
						__snake_case        =						None
						if self.use_labels:
											__snake_case        =						ids_tensor([self.batch_size]   ,     self.type_sequence_label_size						)
											__snake_case        =						ids_tensor([self.batch_size, self.seq_length]   ,     self.num_labels						)
											__snake_case        =						ids_tensor([self.batch_size]   ,     self.num_choices						)

						__snake_case        =						self.get_config()

						return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




	def 				a_		( self	:						str						)       ->			str:
						return DPRConfig(
						    projection_dim=self.projection_dim   ,     vocab_size=self.vocab_size   ,     hidden_size=self.hidden_size   ,     num_hidden_layers=self.num_hidden_layers   ,     num_attention_heads=self.num_attention_heads   ,     intermediate_size=self.intermediate_size   ,     hidden_act=self.hidden_act   ,     hidden_dropout_prob=self.hidden_dropout_prob   ,     attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,     max_position_embeddings=self.max_position_embeddings   ,     type_vocab_size=self.type_vocab_size   ,     initializer_range=self.initializer_range   ,     )




	def 				a_		( self	:						Optional[Any]   ,     lowercase	:						str   ,     lowercase	:						Optional[Any]   ,     lowercase	:						str   ,     lowercase	:						Dict   ,     lowercase	:						Union[str, Any]   ,     lowercase	:						Any   ,     lowercase	:						List[str]						)       ->			Optional[Any]:
						__snake_case        =						DPRContextEncoder(config=lowercase						)
						model.to(lowercase						)
						model.eval()
						__snake_case        =						model(lowercase   ,     attention_mask=lowercase   ,     token_type_ids=lowercase						)
						__snake_case        =						model(lowercase   ,     token_type_ids=lowercase						)
						__snake_case        =						model(lowercase						)
						self.parent.assertEqual(result.pooler_output.shape   ,     (self.batch_size, self.projection_dim or self.hidden_size)						)




	def 				a_		( self	:						Tuple   ,     lowercase	:						Tuple   ,     lowercase	:						Union[str, Any]   ,     lowercase	:						List[str]   ,     lowercase	:						Optional[int]   ,     lowercase	:						Any   ,     lowercase	:						Tuple   ,     lowercase	:						Union[str, Any]						)       ->			Optional[int]:
						__snake_case        =						DPRQuestionEncoder(config=lowercase						)
						model.to(lowercase						)
						model.eval()
						__snake_case        =						model(lowercase   ,     attention_mask=lowercase   ,     token_type_ids=lowercase						)
						__snake_case        =						model(lowercase   ,     token_type_ids=lowercase						)
						__snake_case        =						model(lowercase						)
						self.parent.assertEqual(result.pooler_output.shape   ,     (self.batch_size, self.projection_dim or self.hidden_size)						)




	def 				a_		( self	:						str   ,     lowercase	:						List[Any]   ,     lowercase	:						int   ,     lowercase	:						Optional[Any]   ,     lowercase	:						Any   ,     lowercase	:						Tuple   ,     lowercase	:						Optional[Any]   ,     lowercase	:						int						)       ->			str:
						__snake_case        =						DPRReader(config=lowercase						)
						model.to(lowercase						)
						model.eval()
						__snake_case        =						model(
						    lowercase   ,     attention_mask=lowercase   ,     )

						self.parent.assertEqual(result.start_logits.shape   ,     (self.batch_size, self.seq_length)						)
						self.parent.assertEqual(result.end_logits.shape   ,     (self.batch_size, self.seq_length)						)
						self.parent.assertEqual(result.relevance_logits.shape   ,     (self.batch_size,)						)




	def 				a_		( self	:						str						)       ->			Optional[Any]:
						__snake_case        =						self.prepare_config_and_inputs()
						(
						    (
						    __snake_case
						)   ,   (
						    __snake_case
						)   ,   (
						    __snake_case
						)   ,   (
						    __snake_case
						)   ,   (
						    __snake_case
						)   ,   (
						    __snake_case
						)   ,   (
						    __snake_case
						)   ,   
						)        =						config_and_inputs
						__snake_case        =						{"""input_ids""": input_ids}
						return config, inputs_dict

@require_torch
class      A__   (   lowercase_  ,						lowercase_  ,						unittest.TestCase  ):
	A_				:					Dict      =   (
	    (
	        DPRContextEncoder,
	        DPRQuestionEncoder,
	        DPRReader,
	    )
	    if is_torch_available()
	    else ()
	)
	A_				:					Any      =   {"""feature-extraction""": DPRQuestionEncoder} if is_torch_available() else {}

	A_				:					List[Any]      =   False
	A_				:					Union[str, Any]      =   False  # why?
	A_				:					Any      =   False
	A_				:					List[str]      =   False




	def 				a_		( self	:						Union[str, Any]						)       ->			Dict:
						__snake_case        =						DPRModelTester(self						)
						__snake_case        =						ConfigTester(self   ,     config_class=lowercase   ,     hidden_size=37						)




	def 				a_		( self	:						Optional[int]						)       ->			Optional[int]:
						self.config_tester.run_common_tests()




	def 				a_		( self	:						Dict						)       ->			List[str]:
						__snake_case        =						self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_context_encoder(*lowercase						)




	def 				a_		( self	:						Any						)       ->			Tuple:
						__snake_case        =						self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_question_encoder(*lowercase						)




	def 				a_		( self	:						Any						)       ->			Optional[int]:
						__snake_case        =						self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_reader(*lowercase						)




	def 				a_		( self	:						Tuple						)       ->			str:
						__snake_case        =						self.model_tester.prepare_config_and_inputs()[0]

						__snake_case        =						DPRQuestionEncoder(config=lowercase						)
						model.to(lowercase						)
						model.eval()

						with tempfile.TemporaryDirectory() as tmp_dirname:
											model.save_pretrained(lowercase						)
											__snake_case        =						DPRQuestionEncoder.from_pretrained(lowercase   ,     projection_dim=512						)

						self.assertIsNotNone(lowercase						)




	@slow
	def 				a_		( self	:						Any						)       ->			Optional[Any]:
						for model_name in DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											__snake_case        =						DPRContextEncoder.from_pretrained(lowercase						)
											self.assertIsNotNone(lowercase						)

						for model_name in DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											__snake_case        =						DPRContextEncoder.from_pretrained(lowercase						)
											self.assertIsNotNone(lowercase						)

						for model_name in DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											__snake_case        =						DPRQuestionEncoder.from_pretrained(lowercase						)
											self.assertIsNotNone(lowercase						)

						for model_name in DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											__snake_case        =						DPRReader.from_pretrained(lowercase						)
											self.assertIsNotNone(lowercase						)

@require_torch
class      A__   (   unittest.TestCase  ):




	@slow
	def 				a_		( self	:						List[str]						)       ->			Union[str, Any]:
						__snake_case        =						DPRQuestionEncoder.from_pretrained("""facebook/dpr-question_encoder-single-nq-base"""   ,     return_dict=lowercase						)
						model.to(lowercase						)

						__snake_case        =						torch.tensor(
						    [[101, 7_592, 1_010, 2_003, 2_026, 3_899, 10_140, 1_029, 102]]   ,     dtype=torch.long   ,     device=lowercase						)  # [CLS] hello, is my dog cute? [SEP]
						__snake_case        =						model(lowercase						)[0]  # embedding shape = (1, 768)
						# compare the actual values for a slice.
						__snake_case        =						torch.tensor(
						    [
						        [
						            0.03_23_62_53,
						            0.12_75_33_35,
						            0.16_81_85_09,
						            0.00_27_97_86,
						            0.3_89_69_33,
						            0.24_26_49_45,
						            0.2_17_89_71,
						            -0.02_33_52_27,
						            -0.08_48_19_59,
						            -0.14_32_41_17,
						        ]
						    ]   ,     dtype=torch.float   ,     device=lowercase   ,     )
						self.assertTrue(torch.allclose(output[:, :10]   ,     lowercase   ,     atol=1e-4						)						)




	@slow
	def 				a_		( self	:						Tuple						)       ->			int:
						__snake_case        =						DPRReaderTokenizer.from_pretrained("""facebook/dpr-reader-single-nq-base"""						)
						__snake_case        =						DPRReader.from_pretrained("""facebook/dpr-reader-single-nq-base"""						)
						model.to(lowercase						)

						__snake_case        =						tokenizer(
						    questions="""What is love ?"""   ,     titles="""Haddaway"""   ,     texts="""What Is Love is a song recorded by the artist Haddaway"""   ,     padding=lowercase   ,     return_tensors="""pt"""   ,     )
						encoded_inputs.to(lowercase						)

						__snake_case        =						model(**lowercase						)

						# compare the actual values for a slice.
						__snake_case        =						torch.tensor(
						    [[-10.30_05, -10.77_65, -11.48_72, -11.68_41, -11.93_12, -10.30_02, -9.85_44, -11.73_78, -12.08_21, -10.29_75]]   ,     dtype=torch.float   ,     device=lowercase   ,     )

						__snake_case        =						torch.tensor(
						    [[-11.06_84, -11.70_41, -11.53_97, -10.34_65, -10.87_91, -6.84_43, -11.99_59, -11.03_64, -10.00_96, -6.84_05]]   ,     dtype=torch.float   ,     device=lowercase   ,     )
						self.assertTrue(torch.allclose(outputs.start_logits[:, :10]   ,     lowercase   ,     atol=1e-4						)						)
						self.assertTrue(torch.allclose(outputs.end_logits[:, :10]   ,     lowercase   ,     atol=1e-4						)						)


