





import argparse
import gc
import os
import threading

import evaluate
import psutil
import torch
from datasets import load_dataset
from torch.distributed.fsdp.fully_sharded_data_parallel import FullOptimStateDictConfig, FullStateDictConfig
from torch.utils.data import DataLoader
from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    get_linear_schedule_with_warmup,
    set_seed,
)

from accelerate import Accelerator, DistributedType, FullyShardedDataParallelPlugin


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#   - FSDP
#
# This example also demonstrates the checkpointing and sharding capabilities
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


snake_case__   :				Any						=	16
snake_case__   :				Optional[Any]						=	32



def _a							(      lowerCamelCase:  Tuple					)      ->		Tuple:







      '''simple docstring'''






      return int(x / 2**20					)





class A_   :




   def __enter__(self      :List[str]					)->   Union[str, Any]:
         gc.collect()
         torch.cuda.empty_cache()
         torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
         __A			  =							torch.cuda.memory_allocated()
         __A			  =							psutil.Process()

         __A			  =							self.cpu_mem_used()
         __A			  =							True
         __A			  =							threading.Thread(target=self.peak_monitor_func					)
         __A			  =							True
         peak_monitor_thread.start()
         return self




   def _lowerCAmelCase (self      :Optional[Any]					)->   int:
         return self.process.memory_info().rss




   def _lowerCAmelCase (self      :str					)->   str:
         __A			  =							-1

         while True:
               __A			  =							max(self.cpu_mem_used()       ,  self.cpu_peak					)

               # can't sleep or will not catch the peak right (this comment is here on purpose)
               # time.sleep(0.001) # 1msec

               if not self.peak_monitoring:
                     break






   def __exit__(self      :Tuple       ,  *_UpperCamelCase      :int					)->   Optional[int]:
         __A			  =							False

         gc.collect()
         torch.cuda.empty_cache()
         __A			  =							torch.cuda.memory_allocated()
         __A			  =							torch.cuda.max_memory_allocated()
         __A			  =							bamb(self.end - self.begin					)
         __A			  =							bamb(self.peak - self.begin					)

         __A			  =							self.cpu_mem_used()
         __A			  =							bamb(self.cpu_end - self.cpu_begin					)
         __A			  =							bamb(self.cpu_peak - self.cpu_begin					)
         # print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")


# For testing only
if os.environ.get('TESTING_MOCKED_DATALOADERS', None) == "1":
       from accelerate.test_utils.training import mocked_dataloaders

       snake_case__   :				int						=	mocked_dataloaders  # noqa: F811



def _a							(      lowerCamelCase:  str   ,							lowerCamelCase:  Dict					)      ->		Optional[Any]:







      '''simple docstring'''






      if os.environ.get('''TESTING_MOCKED_DATALOADERS'''   ,							lowerCamelCase_					) == "1":
            __A			  =							2

      # New Code #
      # Pass the advanced FSDP settings not part of the accelerate config by creating fsdp_plugin
      __A			  =							FullyShardedDataParallelPlugin(
          state_dict_config=FullStateDictConfig(offload_to_cpu=lowerCamelCase_   ,							ranka_only=lowerCamelCase_					)   ,							optim_state_dict_config=FullOptimStateDictConfig(offload_to_cpu=lowerCamelCase_   ,							ranka_only=lowerCamelCase_					)   ,							)

      # Initialize accelerator
      if args.with_tracking:
            __A			  =							Accelerator(
                cpu=args.cpu   ,							mixed_precision=args.mixed_precision   ,							log_with='''wandb'''   ,							project_dir=args.logging_dir   ,							fsdp_plugin=lowerCamelCase_   ,							)
      else:
            __A			  =							Accelerator(fsdp_plugin=lowerCamelCase_					)
      accelerator.print(accelerator.distributed_type					)

      if hasattr(args.checkpointing_steps   ,							'''isdigit'''					):
            if args.checkpointing_steps == "epoch":
                  __A			  =							args.checkpointing_steps
            elif args.checkpointing_steps.isdigit():
                  __A			  =							int(args.checkpointing_steps					)
            else:
                  raise ValueError(
                      F"""Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed."""					)
      else:
            __A			  =							None
      # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
      __A			  =							config['''lr''']
      __A			  =							int(config['''num_epochs''']					)
      __A			  =							int(config['''seed''']					)
      __A			  =							int(config['''batch_size''']					)

      # We need to initialize the trackers we use, and also store our configuration
      if args.with_tracking:
            __A			  =							vars(lowerCamelCase_					)
            accelerator.init_trackers('''fsdp_glue_no_trainer'''   ,							lowerCamelCase_					)

      __A			  =							AutoTokenizer.from_pretrained(args.model_name_or_path					)
      __A			  =							load_dataset('''glue'''   ,							'''mrpc'''					)
      __A			  =							evaluate.load('''glue'''   ,							'''mrpc'''					)

      def tokenize_function(lowerCamelCase:  List[str]					):
            # max_length=None => use the model max length (it's actually the default)
            __A			  =							tokenizer(examples['''sentence1''']   ,							examples['''sentence2''']   ,							truncation=lowerCamelCase_   ,							max_length=lowerCamelCase_					)
            return outputs

      # Apply the method we just defined to all the examples in all the splits of the dataset
      # starting with the main process first:
      with accelerator.main_process_first():
            __A			  =							datasets.map(
                lowerCamelCase_   ,							batched=lowerCamelCase_   ,							remove_columns=['''idx''', '''sentence1''', '''sentence2''']   ,							)

      # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
      # transformers library
      __A			  =							tokenized_datasets.rename_column('''label'''   ,							'''labels'''					)

      # If the batch size is too big we use gradient accumulation
      __A			  =							1
      if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
            __A			  =							batch_size // MAX_GPU_BATCH_SIZE
            __A			  =							MAX_GPU_BATCH_SIZE

      def collate_fn(lowerCamelCase:  Dict					):
            # On TPU it's best to pad everything to the same length or training will be very slow.
            __A			  =							1_28 if accelerator.distributed_type == DistributedType.TPU else None
            # When using mixed precision we want round multiples of 8/16
            if accelerator.mixed_precision == "fp8":
                  __A			  =							16
            elif accelerator.mixed_precision != "no":
                  __A			  =							8
            else:
                  __A			  =							None

            return tokenizer.pad(
                lowerCamelCase_   ,							padding='''longest'''   ,							max_length=lowerCamelCase_   ,							pad_to_multiple_of=lowerCamelCase_   ,							return_tensors='''pt'''   ,							)

      # Instantiate dataloaders.
      __A			  =							DataLoader(
          tokenized_datasets['''train''']   ,							shuffle=lowerCamelCase_   ,							collate_fn=lowerCamelCase_   ,							batch_size=lowerCamelCase_					)
      __A			  =							DataLoader(
          tokenized_datasets['''validation''']   ,							shuffle=lowerCamelCase_   ,							collate_fn=lowerCamelCase_   ,							batch_size=lowerCamelCase_					)

      set_seed(lowerCamelCase_					)

      # Instantiate the model (we build the model here so that the seed also control new weights initialization)
      __A			  =							AutoModelForSequenceClassification.from_pretrained(
          args.model_name_or_path   ,							return_dict=lowerCamelCase_   ,							low_cpu_mem_usage=lowerCamelCase_					)

      # New Code #
      # For FSDP feature, it is highly recommended and efficient to prepare the model before creating optimizer
      __A			  =							accelerator.prepare(lowerCamelCase_					)
      accelerator.print(lowerCamelCase_					)

      # Instantiate optimizer
      # New Code #
      # For FSDP feature, at present it doesn't support multiple parameter groups,
      # so we need to create a single parameter group for the whole model
      __A			  =							torch.optim.AdamW(params=model.parameters()   ,							lr=lowerCamelCase_   ,							weight_decay=2e-4					)

      # Instantiate scheduler
      __A			  =							get_linear_schedule_with_warmup(
          optimizer=lowerCamelCase_   ,							num_warmup_steps=10   ,							num_training_steps=(len(lowerCamelCase_					) * num_epochs) // gradient_accumulation_steps   ,							)

      # New Code #
      # For FSDP feature, prepare everything except the model as we have already prepared the model
      # before creating the optimizer
      # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
      # prepare method.
      __A  ,		__A  ,		__A  ,		__A			  =							accelerator.prepare(
          lowerCamelCase_   ,							lowerCamelCase_   ,							lowerCamelCase_   ,							lowerCamelCase_					)

      __A			  =							0

      # Potentially load in the weights and states from a previous save
      if args.resume_from_checkpoint:
            if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
                  accelerator.print(F"""Resumed from checkpoint: {args.resume_from_checkpoint}"""					)
                  accelerator.load_state(args.resume_from_checkpoint					)
                  __A			  =							os.path.basename(args.resume_from_checkpoint					)
            else:
                  # Get the most recent checkpoint
                  __A			  =							[f.name for f in os.scandir(os.getcwd()					) if f.is_dir()]
                  dirs.sort(key=os.path.getctime					)
                  __A			  =							dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
            # Extract `epoch_{i}` or `step_{i}`
            __A			  =							os.path.splitext(lowerCamelCase_					)[0]

            if "epoch" in training_difference:
                  num_epochs -= int(training_difference.replace('''epoch_'''   ,							''''''					)					)
                  __A			  =							None
            else:
                  __A			  =							int(training_difference.replace('''step_'''   ,							''''''					)					)
                  num_epochs -= resume_step // len(lowerCamelCase_					)
                  # If resuming by step, we also need to know exactly how far into the DataLoader we went
                  __A			  =							(num_epochs * len(lowerCamelCase_					)) - resume_step

    # Now we train the model
      for epoch in range(lowerCamelCase_					):
            # New Code #
            # context manager to track the peak memory usage during the training epoch
            with TorchTracemalloc() as tracemalloc:
                  model.train()
                  if args.with_tracking:
                        __A			  =							0
                  for step, batch in enumerate(lowerCamelCase_					):
                        # We need to skip steps until we reach the resumed step
                        if args.resume_from_checkpoint and epoch == 0:
                              if resume_step is not None and step < resume_step:
                                    pass
                # We could avoid this line since we set the accelerator with `device_placement=True`.
                        batch.to(accelerator.device					)
                        __A			  =							model(**lowerCamelCase_					)
                        __A			  =							outputs.loss
                        __A			  =							loss / gradient_accumulation_steps
                        # We keep track of the loss at each epoch
                        if args.with_tracking:
                              total_loss += loss.detach().float()
                        accelerator.backward(lowerCamelCase_					)
                        if step % gradient_accumulation_steps == 0:
                              optimizer.step()
                              lr_scheduler.step()
                              optimizer.zero_grad()
                              # accelerator.print(lr_scheduler.get_lr())

                        overall_step += 1

                        if isinstance(lowerCamelCase_   ,							lowerCamelCase_					):
                              __A			  =							F"""step_{overall_step}"""
                              if overall_step % checkpointing_steps == 0:
                                    if args.output_dir is not None:
                                          __A			  =							os.path.join(args.output_dir   ,							lowerCamelCase_					)
                                    accelerator.save_state(lowerCamelCase_					)
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
            accelerator.print('''Memory before entering the train : {}'''.format(bamb(tracemalloc.begin					)					)					)
            accelerator.print('''Memory consumed at the end of the train (end-begin): {}'''.format(tracemalloc.used					)					)
            accelerator.print('''Peak Memory consumed during the train (max-begin): {}'''.format(tracemalloc.peaked					)					)
            accelerator.print(
                '''Total Peak Memory consumed during the train (max): {}'''.format(
                    tracemalloc.peaked + bamb(tracemalloc.begin					)					)					)
            # Logging the peak memory usage of the GPU to the tracker
            if args.with_tracking:
                  accelerator.log(
                      {
                          '''train_total_peak_memory''': tracemalloc.peaked + bamb(tracemalloc.begin					),
                      }   ,							step=lowerCamelCase_   ,							)

            # New Code #
            # context manager to track the peak memory usage during the evaluation
            with TorchTracemalloc() as tracemalloc:
                  model.eval()
                  for step, batch in enumerate(lowerCamelCase_					):
                        # We could avoid this line since we set the accelerator with `device_placement=True`.
                        batch.to(accelerator.device					)
                        with torch.no_grad():
                              __A			  =							model(**lowerCamelCase_					)
                        __A			  =							outputs.logits.argmax(dim=-1					)
                        __A  ,		__A			  =							accelerator.gather_for_metrics((predictions, batch['''labels'''])					)
                        metric.add_batch(
                            predictions=lowerCamelCase_   ,							references=lowerCamelCase_   ,							)

                  __A			  =							metric.compute()
                  # Use accelerator.print to print only on the main process.
                  accelerator.print(F"""epoch {epoch}:"""   ,							lowerCamelCase_					)
                  if args.with_tracking:
                        accelerator.log(
                            {
                                '''accuracy''': eval_metric['''accuracy'''],
                                '''f1''': eval_metric['''f1'''],
                                '''train_loss''': total_loss.item() / len(lowerCamelCase_					),
                            }   ,							step=lowerCamelCase_   ,							)

                  if checkpointing_steps == "epoch":
                        __A			  =							F"""epoch_{epoch}"""
                        if args.output_dir is not None:
                              __A			  =							os.path.join(args.output_dir   ,							lowerCamelCase_					)
                        accelerator.save_state(lowerCamelCase_					)
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
            accelerator.print('''Memory before entering the eval : {}'''.format(bamb(tracemalloc.begin					)					)					)
            accelerator.print('''Memory consumed at the end of the eval (end-begin): {}'''.format(tracemalloc.used					)					)
            accelerator.print('''Peak Memory consumed during the eval (max-begin): {}'''.format(tracemalloc.peaked					)					)
            accelerator.print(
                '''Total Peak Memory consumed during the eval (max): {}'''.format(tracemalloc.peaked + bamb(tracemalloc.begin					)					)					)
            # Logging the peak memory usage of the GPU to the tracker
            if args.with_tracking:
                  accelerator.log(
                      {
                          '''eval_total_peak_memory''': tracemalloc.peaked + bamb(tracemalloc.begin					),
                      }   ,							step=lowerCamelCase_   ,							)

      if args.with_tracking:
            accelerator.end_training()



def _a							(      )      ->		Optional[Any]:







      '''simple docstring'''






      __A			  =							argparse.ArgumentParser(description='''Simple example of training script.'''					)
      parser.add_argument(
          '''--mixed_precision'''   ,							type=lowerCamelCase_   ,							default=lowerCamelCase_   ,							choices=['''no''', '''fp16''', '''bf16''', '''fp8''']   ,							help='''Whether to use mixed precision. Choose'''
          '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
          '''and an Nvidia Ampere GPU.'''   ,							)
      parser.add_argument('''--cpu'''   ,							action='''store_true'''   ,							help='''If passed, will train on the CPU.'''					)
      parser.add_argument(
          '''--checkpointing_steps'''   ,							type=lowerCamelCase_   ,							default=lowerCamelCase_   ,							help='''Whether the various states should be saved at the end of every n steps, or \'epoch\' for each epoch.'''   ,							)
      parser.add_argument(
          '''--resume_from_checkpoint'''   ,							type=lowerCamelCase_   ,							default=lowerCamelCase_   ,							help='''If the training should continue from a checkpoint folder.'''   ,							)
      parser.add_argument(
          '''--with_tracking'''   ,							action='''store_true'''   ,							help='''Whether to load in all available experiment trackers from the environment and use them for logging.'''   ,							)
      parser.add_argument(
          '''--output_dir'''   ,							type=lowerCamelCase_   ,							default='''.'''   ,							help='''Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'''   ,							)
      parser.add_argument(
          '''--logging_dir'''   ,							type=lowerCamelCase_   ,							default='''logs'''   ,							help='''Location on where to store experiment tracking logs`'''   ,							)
      parser.add_argument(
          '''--model_name_or_path'''   ,							type=lowerCamelCase_   ,							help='''Path to pretrained model or model identifier from huggingface.co/models.'''   ,							required=lowerCamelCase_   ,							)
      __A			  =							parser.parse_args()
      __A			  =							{'''lr''': 2e-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
      training_function(lowerCamelCase_   ,							lowerCamelCase_					)


if __name__ == "__main__":
       main()






import tempfile
import unittest

from transformers import DPRConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import DPRContextEncoder, DPRQuestionEncoder, DPRReader, DPRReaderTokenizer
			from transformers.models.dpr.modeling_dpr import (
			    DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
			    DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
			    DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
			)

class 						_SCREAMING_SNAKE_CASE				:






		"""simple docstring"""







		def __init__(      self     :					List[Any]	,       __SCREAMING_SNAKE_CASE     :					List[Any]	,       __SCREAMING_SNAKE_CASE     :					List[Any]=13	,       __SCREAMING_SNAKE_CASE     :					str=7	,       __SCREAMING_SNAKE_CASE     :					Any=False	,       __SCREAMING_SNAKE_CASE     :					Dict=True	,       __SCREAMING_SNAKE_CASE     :					Any=True	,       __SCREAMING_SNAKE_CASE     :					List[str]=True	,       __SCREAMING_SNAKE_CASE     :					Union[str, Any]=99	,       __SCREAMING_SNAKE_CASE     :					Dict=32	,       __SCREAMING_SNAKE_CASE     :					Union[str, Any]=5	,       __SCREAMING_SNAKE_CASE     :					Dict=4	,       __SCREAMING_SNAKE_CASE     :					Tuple=37	,       __SCREAMING_SNAKE_CASE     :					List[str]="gelu"	,       __SCREAMING_SNAKE_CASE     :					List[str]=0.1	,       __SCREAMING_SNAKE_CASE     :					Optional[Any]=0.1	,       __SCREAMING_SNAKE_CASE     :					List[Any]=512	,       __SCREAMING_SNAKE_CASE     :					Optional[int]=16	,       __SCREAMING_SNAKE_CASE     :					Any=2	,       __SCREAMING_SNAKE_CASE     :					Dict=0.0_2	,       __SCREAMING_SNAKE_CASE     :					Union[str, Any]=3	,       __SCREAMING_SNAKE_CASE     :					int=4	,       __SCREAMING_SNAKE_CASE     :					Optional[Any]=None	,       __SCREAMING_SNAKE_CASE     :					List[Any]=0	,       ):






					"""simple docstring"""
					_snake_case														=   parent
					_snake_case														=   batch_size
					_snake_case														=   seq_length
					_snake_case														=   is_training
					_snake_case														=   use_input_mask
					_snake_case														=   use_token_type_ids
					_snake_case														=   use_labels
					_snake_case														=   vocab_size
					_snake_case														=   hidden_size
					_snake_case														=   num_hidden_layers
					_snake_case														=   num_attention_heads
					_snake_case														=   intermediate_size
					_snake_case														=   hidden_act
					_snake_case														=   hidden_dropout_prob
					_snake_case														=   attention_probs_dropout_prob
					_snake_case														=   max_position_embeddings
					_snake_case														=   type_vocab_size
					_snake_case														=   type_sequence_label_size
					_snake_case														=   initializer_range
					_snake_case														=   num_labels
					_snake_case														=   num_choices
					_snake_case														=   scope
					_snake_case														=   projection_dim



		def 					_a					(      self     :					Tuple ):






					"""simple docstring"""
					_snake_case														=   ids_tensor([self.batch_size, self.seq_length]	,       self.vocab_size )

					_snake_case														=   None
					if self.use_input_mask:
								_snake_case														=   random_attention_mask([self.batch_size, self.seq_length] )

					_snake_case														=   None
					if self.use_token_type_ids:
								_snake_case														=   ids_tensor([self.batch_size, self.seq_length]	,       self.type_vocab_size )

					_snake_case														=   None
					_snake_case														=   None
					_snake_case														=   None
					if self.use_labels:
								_snake_case														=   ids_tensor([self.batch_size]	,       self.type_sequence_label_size )
								_snake_case														=   ids_tensor([self.batch_size, self.seq_length]	,       self.num_labels )
								_snake_case														=   ids_tensor([self.batch_size]	,       self.num_choices )

					_snake_case														=   self.get_config()

					return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



		def 					_a					(      self     :					List[Any] ):






					"""simple docstring"""
					return DPRConfig(
					    projection_dim=self.projection_dim	,       vocab_size=self.vocab_size	,       hidden_size=self.hidden_size	,       num_hidden_layers=self.num_hidden_layers	,       num_attention_heads=self.num_attention_heads	,       intermediate_size=self.intermediate_size	,       hidden_act=self.hidden_act	,       hidden_dropout_prob=self.hidden_dropout_prob	,       attention_probs_dropout_prob=self.attention_probs_dropout_prob	,       max_position_embeddings=self.max_position_embeddings	,       type_vocab_size=self.type_vocab_size	,       initializer_range=self.initializer_range	,       )



		def 					_a					(      self     :					List[Any]	,       __SCREAMING_SNAKE_CASE     :					Union[str, Any]	,       __SCREAMING_SNAKE_CASE     :					List[str]	,       __SCREAMING_SNAKE_CASE     :					List[str]	,       __SCREAMING_SNAKE_CASE     :					List[Any]	,       __SCREAMING_SNAKE_CASE     :					Any	,       __SCREAMING_SNAKE_CASE     :					List[Any]	,       __SCREAMING_SNAKE_CASE     :					List[str] ):






					"""simple docstring"""
					_snake_case														=   DPRContextEncoder(config=__SCREAMING_SNAKE_CASE )
					model.to(__SCREAMING_SNAKE_CASE )
					model.eval()
					_snake_case														=   model(__SCREAMING_SNAKE_CASE	,       attention_mask=__SCREAMING_SNAKE_CASE	,       token_type_ids=__SCREAMING_SNAKE_CASE )
					_snake_case														=   model(__SCREAMING_SNAKE_CASE	,       token_type_ids=__SCREAMING_SNAKE_CASE )
					_snake_case														=   model(__SCREAMING_SNAKE_CASE )
					self.parent.assertEqual(result.pooler_output.shape	,       (self.batch_size, self.projection_dim or self.hidden_size) )



		def 					_a					(      self     :					str	,       __SCREAMING_SNAKE_CASE     :					Optional[Any]	,       __SCREAMING_SNAKE_CASE     :					List[str]	,       __SCREAMING_SNAKE_CASE     :					List[str]	,       __SCREAMING_SNAKE_CASE     :					List[str]	,       __SCREAMING_SNAKE_CASE     :					Dict	,       __SCREAMING_SNAKE_CASE     :					Optional[Any]	,       __SCREAMING_SNAKE_CASE     :					str ):






					"""simple docstring"""
					_snake_case														=   DPRQuestionEncoder(config=__SCREAMING_SNAKE_CASE )
					model.to(__SCREAMING_SNAKE_CASE )
					model.eval()
					_snake_case														=   model(__SCREAMING_SNAKE_CASE	,       attention_mask=__SCREAMING_SNAKE_CASE	,       token_type_ids=__SCREAMING_SNAKE_CASE )
					_snake_case														=   model(__SCREAMING_SNAKE_CASE	,       token_type_ids=__SCREAMING_SNAKE_CASE )
					_snake_case														=   model(__SCREAMING_SNAKE_CASE )
					self.parent.assertEqual(result.pooler_output.shape	,       (self.batch_size, self.projection_dim or self.hidden_size) )



		def 					_a					(      self     :					List[str]	,       __SCREAMING_SNAKE_CASE     :					int	,       __SCREAMING_SNAKE_CASE     :					List[Any]	,       __SCREAMING_SNAKE_CASE     :					int	,       __SCREAMING_SNAKE_CASE     :					Union[str, Any]	,       __SCREAMING_SNAKE_CASE     :					Any	,       __SCREAMING_SNAKE_CASE     :					Optional[int]	,       __SCREAMING_SNAKE_CASE     :					Any ):






					"""simple docstring"""
					_snake_case														=   DPRReader(config=__SCREAMING_SNAKE_CASE )
					model.to(__SCREAMING_SNAKE_CASE )
					model.eval()
					_snake_case														=   model(
					    __SCREAMING_SNAKE_CASE	,       attention_mask=__SCREAMING_SNAKE_CASE	,       )

					self.parent.assertEqual(result.start_logits.shape	,       (self.batch_size, self.seq_length) )
					self.parent.assertEqual(result.end_logits.shape	,       (self.batch_size, self.seq_length) )
					self.parent.assertEqual(result.relevance_logits.shape	,       (self.batch_size,) )







		def 					_a					(      self     :					Any ):






					"""simple docstring"""
					_snake_case														=   self.prepare_config_and_inputs()
					(
					    (
					    _snake_case
					)							,			(
					    _snake_case
					)							,			(
					    _snake_case
					)							,			(
					    _snake_case
					)							,			(
					    _snake_case
					)							,			(
					    _snake_case
					)							,			(
					    _snake_case
					)							,			
					)														=   config_and_inputs
					_snake_case														=   {"input_ids": input_ids}
					return config, inputs_dict

@require_torch
class 						_SCREAMING_SNAKE_CASE				(  _lowercase   ,  _lowercase   ,  unittest.TestCase				):






		"""simple docstring"""







		A_    			=   (
		    (
		        DPRContextEncoder,
		        DPRQuestionEncoder,
		        DPRReader,
		    )
		    if is_torch_available()
		    else ()
		)
		A_    			=   {'''feature-extraction''': DPRQuestionEncoder} if is_torch_available() else {}

		A_    			=   False
		A_    			=   False  # why?
		A_    			=   False
		A_    			=   False



		def 					_a					(      self     :					Union[str, Any] ):






					"""simple docstring"""
					_snake_case														=   DPRModelTester(self )
					_snake_case														=   ConfigTester(self	,       config_class=__SCREAMING_SNAKE_CASE	,       hidden_size=37 )



		def 					_a					(      self     :					int ):






					"""simple docstring"""
					self.config_tester.run_common_tests()



		def 					_a					(      self     :					List[str] ):






					"""simple docstring"""
					_snake_case														=   self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_context_encoder(*__SCREAMING_SNAKE_CASE )



		def 					_a					(      self     :					List[str] ):






					"""simple docstring"""
					_snake_case														=   self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_question_encoder(*__SCREAMING_SNAKE_CASE )



		def 					_a					(      self     :					Optional[Any] ):






					"""simple docstring"""
					_snake_case														=   self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_reader(*__SCREAMING_SNAKE_CASE )



		def 					_a					(      self     :					Optional[Any] ):






					"""simple docstring"""
					_snake_case														=   self.model_tester.prepare_config_and_inputs()[0]

					_snake_case														=   DPRQuestionEncoder(config=__SCREAMING_SNAKE_CASE )
					model.to(__SCREAMING_SNAKE_CASE )
					model.eval()

					with tempfile.TemporaryDirectory() as tmp_dirname:
								model.save_pretrained(__SCREAMING_SNAKE_CASE )
								_snake_case														=   DPRQuestionEncoder.from_pretrained(__SCREAMING_SNAKE_CASE	,       projection_dim=512 )

					self.assertIsNotNone(__SCREAMING_SNAKE_CASE )







		@slow
		def 					_a					(      self     :					List[str] ):






					"""simple docstring"""
					for model_name in DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								_snake_case														=   DPRContextEncoder.from_pretrained(__SCREAMING_SNAKE_CASE )
								self.assertIsNotNone(__SCREAMING_SNAKE_CASE )

					for model_name in DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								_snake_case														=   DPRContextEncoder.from_pretrained(__SCREAMING_SNAKE_CASE )
								self.assertIsNotNone(__SCREAMING_SNAKE_CASE )

					for model_name in DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								_snake_case														=   DPRQuestionEncoder.from_pretrained(__SCREAMING_SNAKE_CASE )
								self.assertIsNotNone(__SCREAMING_SNAKE_CASE )

					for model_name in DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								_snake_case														=   DPRReader.from_pretrained(__SCREAMING_SNAKE_CASE )
								self.assertIsNotNone(__SCREAMING_SNAKE_CASE )

@require_torch
class 						_SCREAMING_SNAKE_CASE				(  unittest.TestCase				):






		"""simple docstring"""







		@slow
		def 					_a					(      self     :					Dict ):






					"""simple docstring"""
					_snake_case														=   DPRQuestionEncoder.from_pretrained("facebook/dpr-question_encoder-single-nq-base"	,       return_dict=__SCREAMING_SNAKE_CASE )
					model.to(__SCREAMING_SNAKE_CASE )

					_snake_case														=   torch.tensor(
					    [[101, 7_592, 1_010, 2_003, 2_026, 3_899, 10_140, 1_029, 102]]	,       dtype=torch.long	,       device=__SCREAMING_SNAKE_CASE )  # [CLS] hello, is my dog cute? [SEP]
					_snake_case														=   model(__SCREAMING_SNAKE_CASE )[0]  # embedding shape = (1, 768)
					# compare the actual values for a slice.
					_snake_case														=   torch.tensor(
					    [
					        [
					            0.0_3_2_3_6_2_5_3,
					            0.1_2_7_5_3_3_3_5,
					            0.1_6_8_1_8_5_0_9,
					            0.0_0_2_7_9_7_8_6,
					            0.3_8_9_6_9_3_3,
					            0.2_4_2_6_4_9_4_5,
					            0.2_1_7_8_9_7_1,
					            -0.0_2_3_3_5_2_2_7,
					            -0.0_8_4_8_1_9_5_9,
					            -0.1_4_3_2_4_1_1_7,
					        ]
					    ]	,       dtype=torch.float	,       device=__SCREAMING_SNAKE_CASE	,       )
					self.assertTrue(torch.allclose(output[:, :10]	,       __SCREAMING_SNAKE_CASE	,       atol=1e-4 ) )







		@slow
		def 					_a					(      self     :					Optional[int] ):






					"""simple docstring"""
					_snake_case														=   DPRReaderTokenizer.from_pretrained("facebook/dpr-reader-single-nq-base" )
					_snake_case														=   DPRReader.from_pretrained("facebook/dpr-reader-single-nq-base" )
					model.to(__SCREAMING_SNAKE_CASE )

					_snake_case														=   tokenizer(
					    questions="What is love ?"	,       titles="Haddaway"	,       texts="What Is Love is a song recorded by the artist Haddaway"	,       padding=__SCREAMING_SNAKE_CASE	,       return_tensors="pt"	,       )
					encoded_inputs.to(__SCREAMING_SNAKE_CASE )

					_snake_case														=   model(**__SCREAMING_SNAKE_CASE )

					# compare the actual values for a slice.
					_snake_case														=   torch.tensor(
					    [[-1_0.3_0_0_5, -1_0.7_7_6_5, -1_1.4_8_7_2, -1_1.6_8_4_1, -1_1.9_3_1_2, -1_0.3_0_0_2, -9.8_5_4_4, -1_1.7_3_7_8, -1_2.0_8_2_1, -1_0.2_9_7_5]]	,       dtype=torch.float	,       device=__SCREAMING_SNAKE_CASE	,       )

					_snake_case														=   torch.tensor(
					    [[-1_1.0_6_8_4, -1_1.7_0_4_1, -1_1.5_3_9_7, -1_0.3_4_6_5, -1_0.8_7_9_1, -6.8_4_4_3, -1_1.9_9_5_9, -1_1.0_3_6_4, -1_0.0_0_9_6, -6.8_4_0_5]]	,       dtype=torch.float	,       device=__SCREAMING_SNAKE_CASE	,       )
					self.assertTrue(torch.allclose(outputs.start_logits[:, :10]	,       __SCREAMING_SNAKE_CASE	,       atol=1e-4 ) )
					self.assertTrue(torch.allclose(outputs.end_logits[:, :10]	,       __SCREAMING_SNAKE_CASE	,       atol=1e-4 ) )

