




import math
from typing import Optional

import torch
import torch.utils.checkpoint as checkpoint
from torch import nn

from ....modeling_utils import PreTrainedModel
from ....utils import add_start_docstrings, logging
from ...bert.modeling_bert import BertModel
from .configuration_retribert import RetriBertConfig


a_ 		=  logging.get_logger(__name__)

a_ 		=  [
    "yjernite/retribert-base-uncased",
    # See all RetriBert models at https://huggingface.co/models?filter=retribert
]

class 			_lowerCamelCase    (      _SCREAMING_SNAKE_CASE ):

      """simple docstring"""

      lowerCAmelCase__      :			Optional[Any]     =       RetriBertConfig
      lowerCAmelCase__      :			Optional[Any]     =       None
      lowerCAmelCase__      :			Dict     =       "retribert"







      def    snake_case						(  self       :			Dict			,							snake_case       :			Optional[Any]					):
          if isinstance(snake_case			,							nn.Linear					):
              module.weight.data.normal_(mean=0.0			,							std=self.config.initializer_range					)
              if module.bias is not None:
                  module.bias.data.zero_()
          elif isinstance(snake_case			,							nn.Embedding					):
              module.weight.data.normal_(mean=0.0			,							std=self.config.initializer_range					)
              if module.padding_idx is not None:
                  module.weight.data[module.padding_idx].zero_()
          elif isinstance(snake_case			,							nn.LayerNorm					):
              module.bias.data.zero_()
              module.weight.data.fill_(1.0					)







a_ 		=  R"\n\n    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the\n    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads\n    etc.)\n\n    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.\n    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage\n    and behavior.\n\n    Parameters:\n        config ([`RetriBertConfig`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.\n"

@add_start_docstrings(
    "Bert Based model to embed queries or document for document retrieval."				,      _SCREAMING_SNAKE_CASE				,      )
class 			_lowerCamelCase    (      _SCREAMING_SNAKE_CASE ):

      """simple docstring"""

      def __init__(  self       :			List[Any]			,							snake_case       :			RetriBertConfig					):
          super().__init__(snake_case					)
          __UpperCamelCase			=    config.projection_dim

          __UpperCamelCase			=    BertModel(snake_case					)
          __UpperCamelCase			=    None if config.share_encoders else BertModel(snake_case					)
          __UpperCamelCase			=    nn.Dropout(config.hidden_dropout_prob					)
          __UpperCamelCase			=    nn.Linear(config.hidden_size			,							config.projection_dim			,							bias=snake_case					)
          __UpperCamelCase			=    nn.Linear(config.hidden_size			,							config.projection_dim			,							bias=snake_case					)

          __UpperCamelCase			=    nn.CrossEntropyLoss(reduction='''mean'''					)

          # Initialize weights and apply final processing
          self.post_init()







      def    snake_case						(  self       :			Dict			,							snake_case       :			Dict			,							snake_case       :			List[str]			,							snake_case       :			Dict			,							snake_case       :			Any=-1			,							):
          # reproduces BERT forward pass with checkpointing
          if checkpoint_batch_size < 0 or input_ids.shape[0] < checkpoint_batch_size:
              return sent_encoder(snake_case			,							attention_mask=snake_case					)[1]
          else:
              # prepare implicit variables
              __UpperCamelCase			=    input_ids.device
              __UpperCamelCase			=    input_ids.size()
              __UpperCamelCase			=    torch.zeros(snake_case			,							dtype=torch.long			,							device=snake_case					)
              __UpperCamelCase			=    [None] * sent_encoder.config.num_hidden_layers
              __UpperCamelCase			=    sent_encoder.get_extended_attention_mask(
                  snake_case			,							snake_case					)

              # define function for checkpointing
              def partial_encode(*snake_case       :			Optional[Any]					):
                  __UpperCamelCase			=    sent_encoder.encoder(
                      inputs[0]			,							attention_mask=inputs[1]			,							head_mask=snake_case			,							)
                  __UpperCamelCase			=    encoder_outputs[0]
                  __UpperCamelCase			=    sent_encoder.pooler(snake_case					)
                  return pooled_output

              # run embedding layer on everything at once
              __UpperCamelCase			=    sent_encoder.embeddings(
                  input_ids=snake_case			,							position_ids=snake_case			,							token_type_ids=snake_case			,							inputs_embeds=snake_case					)
              # run encoding and pooling on one mini-batch at a time
              __UpperCamelCase			=    []
              for b in range(math.ceil(input_ids.shape[0] / checkpoint_batch_size					)					):
                  __UpperCamelCase			=    embedding_output[b * checkpoint_batch_size : (b + 1) * checkpoint_batch_size]
                  __UpperCamelCase			=    extended_attention_mask[b * checkpoint_batch_size : (b + 1) * checkpoint_batch_size]
                  __UpperCamelCase			=    checkpoint.checkpoint(snake_case			,							snake_case			,							snake_case					)
                  pooled_output_list.append(snake_case					)
              return torch.cat(snake_case			,							dim=0					)







      def    snake_case						(  self       :			Dict			,							snake_case       :			Any			,							snake_case       :			Union[str, Any]=None			,							snake_case       :			List[str]=-1			,							):
          __UpperCamelCase			=    self.embed_sentences_checkpointed(
              snake_case			,							snake_case			,							self.bert_query			,							snake_case			,							)
          return self.project_query(snake_case					)







      def    snake_case						(  self       :			Optional[Any]			,							snake_case       :			Optional[Any]			,							snake_case       :			Optional[int]=None			,							snake_case       :			Optional[Any]=-1			,							):
          __UpperCamelCase			=    self.embed_sentences_checkpointed(
              snake_case			,							snake_case			,							self.bert_query if self.bert_doc is None else self.bert_doc			,							snake_case			,							)
          return self.project_doc(snake_case					)







      def    snake_case						(  self       :			List[str]			,							snake_case       :			torch.LongTensor			,							snake_case       :			Optional[torch.FloatTensor]			,							snake_case       :			torch.LongTensor			,							snake_case       :			Optional[torch.FloatTensor]			,							snake_case       :			int = -1			,							):
          __UpperCamelCase			=    input_ids_query.device
          __UpperCamelCase			=    self.embed_questions(snake_case			,							snake_case			,							snake_case					)
          __UpperCamelCase			=    self.embed_answers(snake_case			,							snake_case			,							snake_case					)
          __UpperCamelCase			=    torch.mm(snake_case			,							a_reps.t()					)
          __UpperCamelCase			=    self.ce_loss(snake_case			,							torch.arange(compare_scores.shape[1]					).to(snake_case					)					)
          __UpperCamelCase			=    self.ce_loss(compare_scores.t()			,							torch.arange(compare_scores.shape[0]					).to(snake_case					)					)
          __UpperCamelCase			=    (loss_qa + loss_aq) / 2
          return loss








import tempfile

import torch

from diffusers import (
    DEISMultistepScheduler,
    DPMSolverMultistepScheduler,
    DPMSolverSinglestepScheduler,
    UniPCMultistepScheduler,
)

from .test_schedulers import SchedulerCommonTest


class 	UpperCamelCase__       (  UpperCAmelCase__):

						'''simple docstring'''



						__a     :    List[Any]												=			(DPMSolverMultistepScheduler,)
						__a     :    Dict												=			(("""num_inference_steps""", 2_5),)


						def    A__       (  self  , **A	)  ->List[Any]:
									UpperCAmelCase__		:int 		=    {
									    'num_train_timesteps': 10_00,
									    'beta_start': 0.0001,
									    'beta_end': 0.02,
									    'beta_schedule': 'linear',
									    'solver_order': 2,
									    'prediction_type': 'epsilon',
									    'thresholding': False,
									    'sample_max_value': 1.0,
									    'algorithm_type': 'dpmsolver++',
									    'solver_type': 'midpoint',
									    'lower_order_final': False,
									    'lambda_min_clipped': -float('inf'	),
									    'variance_type': None,
									}

									config.update(**A	)
									return config


						def    A__       (  self  , A=0  , **A	)  ->Tuple:
									UpperCAmelCase__		:List[str] 		=    dict(self.forward_default_kwargs	)
									UpperCAmelCase__		:Any 		=    kwargs.pop('num_inference_steps'  , A	)
									UpperCAmelCase__		:int 		=    self.dummy_sample
									UpperCAmelCase__		:Optional[int] 		=    0.1 * sample
									UpperCAmelCase__		:Any 		=    [residual + 0.2, residual + 0.15, residual + 0.10]

									for scheduler_class in self.scheduler_classes:
												UpperCAmelCase__		:Optional[int] 		=    self.get_scheduler_config(**A	)
												UpperCAmelCase__		:List[Any] 		=    scheduler_class(**A	)
												scheduler.set_timesteps(A	)
												# copy over dummy past residuals
												UpperCAmelCase__		:Optional[Any] 		=    dummy_past_residuals[: scheduler.config.solver_order]

												with tempfile.TemporaryDirectory() as tmpdirname:
															scheduler.save_config(A	)
															UpperCAmelCase__		:List[Any] 		=    scheduler_class.from_pretrained(A	)
															new_scheduler.set_timesteps(A	)
															# copy over dummy past residuals
															UpperCAmelCase__		:Union[str, Any] 		=    dummy_past_residuals[: new_scheduler.config.solver_order]

												UpperCAmelCase__		,       UpperCAmelCase__		:int 		=    sample, sample
												for t in range(A  , time_step + scheduler.config.solver_order + 1	):
															UpperCAmelCase__		:Dict 		=    scheduler.step(A  , A  , A  , **A	).prev_sample
															UpperCAmelCase__		:Optional[Any] 		=    new_scheduler.step(A  , A  , A  , **A	).prev_sample

															assert torch.sum(torch.abs(output - new_output	)	) < 1e-5, "Scheduler outputs are not identical"


						def    A__       (  self	)  ->List[Any]:
									pass


						def    A__       (  self  , A=0  , **A	)  ->Tuple:
									UpperCAmelCase__		:List[str] 		=    dict(self.forward_default_kwargs	)
									UpperCAmelCase__		:Dict 		=    kwargs.pop('num_inference_steps'  , A	)
									UpperCAmelCase__		:str 		=    self.dummy_sample
									UpperCAmelCase__		:Union[str, Any] 		=    0.1 * sample
									UpperCAmelCase__		:Union[str, Any] 		=    [residual + 0.2, residual + 0.15, residual + 0.10]

									for scheduler_class in self.scheduler_classes:
												UpperCAmelCase__		:List[Any] 		=    self.get_scheduler_config()
												UpperCAmelCase__		:List[str] 		=    scheduler_class(**A	)
												scheduler.set_timesteps(A	)

												# copy over dummy past residuals (must be after setting timesteps)
												UpperCAmelCase__		:Tuple 		=    dummy_past_residuals[: scheduler.config.solver_order]

												with tempfile.TemporaryDirectory() as tmpdirname:
															scheduler.save_config(A	)
															UpperCAmelCase__		:int 		=    scheduler_class.from_pretrained(A	)
															# copy over dummy past residuals
															new_scheduler.set_timesteps(A	)

															# copy over dummy past residual (must be after setting timesteps)
															UpperCAmelCase__		:Optional[int] 		=    dummy_past_residuals[: new_scheduler.config.solver_order]

												UpperCAmelCase__		:Any 		=    scheduler.step(A  , A  , A  , **A	).prev_sample
												UpperCAmelCase__		:Dict 		=    new_scheduler.step(A  , A  , A  , **A	).prev_sample

												assert torch.sum(torch.abs(output - new_output	)	) < 1e-5, "Scheduler outputs are not identical"


						def    A__       (  self  , A=None  , **A	)  ->List[str]:
									if scheduler is None:
												UpperCAmelCase__		:Optional[Any] 		=    self.scheduler_classes[0]
												UpperCAmelCase__		:Optional[Any] 		=    self.get_scheduler_config(**A	)
												UpperCAmelCase__		:Union[str, Any] 		=    scheduler_class(**A	)

									UpperCAmelCase__		:List[Any] 		=    10
									UpperCAmelCase__		:int 		=    self.dummy_model()
									UpperCAmelCase__		:int 		=    self.dummy_sample_deter
									scheduler.set_timesteps(A	)

									for i, t in enumerate(scheduler.timesteps	):
												UpperCAmelCase__		:Union[str, Any] 		=    model(A  , A	)
												UpperCAmelCase__		:str 		=    scheduler.step(A  , A  , A	).prev_sample

									return sample


						def    A__       (  self	)  ->str:
									UpperCAmelCase__		:Tuple 		=    dict(self.forward_default_kwargs	)

									UpperCAmelCase__		:Optional[Any] 		=    kwargs.pop('num_inference_steps'  , A	)

									for scheduler_class in self.scheduler_classes:
												UpperCAmelCase__		:Union[str, Any] 		=    self.get_scheduler_config()
												UpperCAmelCase__		:Optional[Any] 		=    scheduler_class(**A	)

												UpperCAmelCase__		:Dict 		=    self.dummy_sample
												UpperCAmelCase__		:Dict 		=    0.1 * sample

												if num_inference_steps is not None and hasattr(A  , 'set_timesteps'	):
															scheduler.set_timesteps(A	)
												elif num_inference_steps is not None and not hasattr(A  , 'set_timesteps'	):
															UpperCAmelCase__		:List[Any] 		=    num_inference_steps

												# copy over dummy past residuals (must be done after set_timesteps)
												UpperCAmelCase__		:List[Any] 		=    [residual + 0.2, residual + 0.15, residual + 0.10]
												UpperCAmelCase__		:Any 		=    dummy_past_residuals[: scheduler.config.solver_order]

												UpperCAmelCase__		:str 		=    scheduler.timesteps[5]
												UpperCAmelCase__		:str 		=    scheduler.timesteps[6]

												UpperCAmelCase__		:List[str] 		=    scheduler.step(A  , A  , A  , **A	).prev_sample
												UpperCAmelCase__		:List[Any] 		=    scheduler.step(A  , A  , A  , **A	).prev_sample

												self.assertEqual(output_a.shape  , sample.shape	)
												self.assertEqual(output_a.shape  , output_a.shape	)


						def    A__       (  self	)  ->Optional[Any]:
									for timesteps in [25, 50, 1_00, 9_99, 10_00]:
												self.check_over_configs(num_train_timesteps=A	)


						def    A__       (  self	)  ->str:
									self.check_over_configs(thresholding=A	)
									for order in [1, 2, 3]:
												for solver_type in ["midpoint", "heun"]:
															for threshold in [0.5, 1.0, 2.0]:
																		for prediction_type in ["epsilon", "sample"]:
																					self.check_over_configs(
																					    thresholding=A  , prediction_type=A  , sample_max_value=A  , algorithm_type='dpmsolver++'  , solver_order=A  , solver_type=A  , )


						def    A__       (  self	)  ->Optional[int]:
									for prediction_type in ["epsilon", "v_prediction"]:
												self.check_over_configs(prediction_type=A	)


						def    A__       (  self	)  ->List[str]:
									for algorithm_type in ["dpmsolver", "dpmsolver++", "sde-dpmsolver", "sde-dpmsolver++"]:
												for solver_type in ["midpoint", "heun"]:
															for order in [1, 2, 3]:
																		for prediction_type in ["epsilon", "sample"]:
																					if algorithm_type in ["sde-dpmsolver", "sde-dpmsolver++"]:
																								if order == 3:
																											continue
																					else:
																								self.check_over_configs(
																								    solver_order=A  , solver_type=A  , prediction_type=A  , algorithm_type=A  , )
																					UpperCAmelCase__		:Optional[int] 		=    self.full_loop(
																					    solver_order=A  , solver_type=A  , prediction_type=A  , algorithm_type=A  , )
																					assert not torch.isnan(A	).any(), "Samples have nan numbers"


						def    A__       (  self	)  ->List[Any]:
									self.check_over_configs(lower_order_final=A	)
									self.check_over_configs(lower_order_final=A	)


						def    A__       (  self	)  ->Optional[Any]:
									self.check_over_configs(lambda_min_clipped=-float('inf'	)	)
									self.check_over_configs(lambda_min_clipped=-5.1	)


						def    A__       (  self	)  ->Union[str, Any]:
									self.check_over_configs(variance_type=A	)
									self.check_over_configs(variance_type='learned_range'	)


						def    A__       (  self	)  ->List[str]:
									for num_inference_steps in [1, 2, 3, 5, 10, 50, 1_00, 9_99, 10_00]:
												self.check_over_forward(num_inference_steps=A  , time_step=0	)


						def    A__       (  self	)  ->int:
									UpperCAmelCase__		:int 		=    self.full_loop()
									UpperCAmelCase__		:Optional[int] 		=    torch.mean(torch.abs(A	)	)

									assert abs(result_mean.item() - 0.3301	) < 1e-3


						def    A__       (  self	)  ->Any:
									UpperCAmelCase__		:Optional[int] 		=    self.full_loop(thresholding=A  , dynamic_thresholding_ratio=0.87  , sample_max_value=0.5	)
									UpperCAmelCase__		:Any 		=    torch.mean(torch.abs(A	)	)

									assert abs(result_mean.item() - 1.1364	) < 1e-3


						def    A__       (  self	)  ->Union[str, Any]:
									UpperCAmelCase__		:Optional[Any] 		=    self.full_loop(prediction_type='v_prediction'	)
									UpperCAmelCase__		:int 		=    torch.mean(torch.abs(A	)	)

									assert abs(result_mean.item() - 0.2251	) < 1e-3


						def    A__       (  self	)  ->str:
									UpperCAmelCase__		:Optional[int] 		=    self.full_loop(prediction_type='v_prediction'  , use_karras_sigmas=A	)
									UpperCAmelCase__		:Optional[int] 		=    torch.mean(torch.abs(A	)	)

									assert abs(result_mean.item() - 0.2096	) < 1e-3


						def    A__       (  self	)  ->List[Any]:
									# make sure that iterating over schedulers with same config names gives same results
									# for defaults
									UpperCAmelCase__		:Tuple 		=    DPMSolverMultistepScheduler(**self.get_scheduler_config()	)
									UpperCAmelCase__		:Optional[int] 		=    self.full_loop(scheduler=A	)
									UpperCAmelCase__		:List[Any] 		=    torch.mean(torch.abs(A	)	)

									assert abs(result_mean.item() - 0.3301	) < 1e-3

									UpperCAmelCase__		:Dict 		=    DPMSolverSinglestepScheduler.from_config(scheduler.config	)
									UpperCAmelCase__		:List[str] 		=    UniPCMultistepScheduler.from_config(scheduler.config	)
									UpperCAmelCase__		:Union[str, Any] 		=    DEISMultistepScheduler.from_config(scheduler.config	)
									UpperCAmelCase__		:List[str] 		=    DPMSolverMultistepScheduler.from_config(scheduler.config	)

									UpperCAmelCase__		:Dict 		=    self.full_loop(scheduler=A	)
									UpperCAmelCase__		:Dict 		=    torch.mean(torch.abs(A	)	)

									assert abs(result_mean.item() - 0.3301	) < 1e-3


						def    A__       (  self	)  ->Dict:
									UpperCAmelCase__		:Tuple 		=    self.scheduler_classes[0]
									UpperCAmelCase__		:List[str] 		=    self.get_scheduler_config(thresholding=A  , dynamic_thresholding_ratio=0	)
									UpperCAmelCase__		:str 		=    scheduler_class(**A	)

									UpperCAmelCase__		:int 		=    10
									UpperCAmelCase__		:Optional[int] 		=    self.dummy_model()
									UpperCAmelCase__		:List[str] 		=    self.dummy_sample_deter.half()
									scheduler.set_timesteps(A	)

									for i, t in enumerate(scheduler.timesteps	):
												UpperCAmelCase__		:str 		=    model(A  , A	)
												UpperCAmelCase__		:Union[str, Any] 		=    scheduler.step(A  , A  , A	).prev_sample

									assert sample.dtype == torch.floataa


						def    A__       (  self  , **A	)  ->Any:
									for scheduler_class in self.scheduler_classes:
												UpperCAmelCase__		:Optional[int] 		=    self.get_scheduler_config(**A	)
												UpperCAmelCase__		:int 		=    scheduler_class(**A	)

												scheduler.set_timesteps(scheduler.config.num_train_timesteps	)
												assert len(scheduler.timesteps.unique()	) == scheduler.num_inference_steps





