

'''simple docstring'''
import copy

from ...configuration_utils import PretrainedConfig
from ...utils import add_start_docstrings


lowerCamelCase :      List[Any]					 =     r"\n    [`RagConfig`] stores the configuration of a *RagModel*. Configuration objects inherit from [`PretrainedConfig`] and\n    can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information.\n\n    Args:\n        title_sep (`str`, *optional*, defaults to  `\" / \"`):\n            Separator inserted between the title and the text of the retrieved document when calling [`RagRetriever`].\n        doc_sep (`str`, *optional*, defaults to  `\" // \"`):\n            Separator inserted between the text of the retrieved document and the original input when calling\n            [`RagRetriever`].\n        n_docs (`int`, *optional*, defaults to 5):\n            Number of documents to retrieve.\n        max_combined_length (`int`, *optional*, defaults to 300):\n            Max length of contextualized input returned by [`~RagRetriever.__call__`].\n        retrieval_vector_size (`int`, *optional*, defaults to 768):\n            Dimensionality of the document embeddings indexed by [`RagRetriever`].\n        retrieval_batch_size (`int`, *optional*, defaults to 8):\n            Retrieval batch size, defined as the number of queries issues concurrently to the faiss index encapsulated\n            [`RagRetriever`].\n        dataset (`str`, *optional*, defaults to `\"wiki_dpr\"`):\n            A dataset identifier of the indexed dataset in HuggingFace Datasets (list all available datasets and ids\n            using `datasets.list_datasets()`).\n        dataset_split (`str`, *optional*, defaults to `\"train\"`)\n            Which split of the `dataset` to load.\n        index_name (`str`, *optional*, defaults to `\"compressed\"`)\n            The index name of the index associated with the `dataset`. One can choose between `\"legacy\"`, `\"exact\"` and\n            `\"compressed\"`.\n        index_path (`str`, *optional*)\n            The path to the serialized faiss index on disk.\n        passages_path (`str`, *optional*):\n            A path to text passages compatible with the faiss index. Required if using\n            [`~models.rag.retrieval_rag.LegacyIndex`]\n        use_dummy_dataset (`bool`, *optional*, defaults to `False`)\n            Whether to load a \"dummy\" variant of the dataset specified by `dataset`.\n        label_smoothing (`float`, *optional*, defaults to 0.0):\n            Only relevant if `return_loss` is set to `True`. Controls the `epsilon` parameter value for label smoothing\n            in the loss calculation. If set to 0, no label smoothing is performed.\n        do_marginalize (`bool`, *optional*, defaults to `False`):\n            If `True`, the logits are marginalized over all documents by making use of\n            `torch.nn.functional.log_softmax`.\n        reduce_loss (`bool`, *optional*, defaults to `False`):\n            Whether or not to reduce the NLL loss using the `torch.Tensor.sum` operation.\n        do_deduplication (`bool`, *optional*, defaults to `True`):\n            Whether or not to deduplicate the generations from different context documents for a given input. Has to be\n            set to `False` if used while training with distributed backend.\n        exclude_bos_score (`bool`, *optional*, defaults to `False`):\n            Whether or not to disregard the BOS token when computing the loss.\n        output_retrieved(`bool`, *optional*, defaults to `False`):\n            If set to `True`, `retrieved_doc_embeds`, `retrieved_doc_ids`, `context_input_ids` and\n            `context_attention_mask` are returned. See returned tensors for more detail.\n        use_cache (`bool`, *optional*, defaults to `True`):\n            Whether or not the model should return the last key/values attentions (not used by all models).\n        forced_eos_token_id (`int`, *optional*):\n            The id of the token to force as the last generated token when `max_length` is reached. Usually set to\n            `eos_token_id`.\n"







@add_start_docstrings(A__  )
class 							A__    (       A__  ):
  A__		 =				'rag'
  A__		 =				True





  def __init__(		self      :	Optional[Any]							,					_a      :	Union[str, Any]=None							,					_a      :	Optional[Any]=True							,					_a      :	Any=None							,					_a      :	Dict=None							,					_a      :	Optional[Any]=None							,					_a      :	Tuple=None							,					_a      :	List[Any]=None							,					_a      :	Optional[Any]=" / "							,					_a      :	str=" // "							,					_a      :	Tuple=5							,					_a      :	Optional[int]=300							,					_a      :	Optional[Any]=768							,					_a      :	Union[str, Any]=8							,					_a      :	Dict="wiki_dpr"							,					_a      :	Tuple="train"							,					_a      :	Any="compressed"							,					_a      :	Union[str, Any]=None							,					_a      :	Optional[int]=None							,					_a      :	Optional[int]=False							,					_a      :	Any=False							,					_a      :	str=0.0							,					_a      :	Optional[int]=True							,					_a      :	Optional[int]=False							,					_a      :	Optional[int]=False							,					_a      :	int=False							,					_a      :	Any=True							,					_a      :	Union[str, Any]=None							,					**_a      :	List[str]							,					)	->    str:





       '''simple docstring'''






       super().__init__(
           bos_token_id=_a							,					pad_token_id=_a							,					eos_token_id=_a							,					decoder_start_token_id=_a							,					forced_eos_token_id=_a							,					is_encoder_decoder=_a							,					prefix=_a							,					vocab_size=_a							,					**_a							,					)
       assert (
           "question_encoder" in kwargs and "generator" in kwargs
       ), "Config has to be initialized with question_encoder and generator config"
       _SCREAMING_SNAKE_CASE							    =kwargs.pop('question_encoder' )
       _SCREAMING_SNAKE_CASE							    =question_encoder_config.pop('model_type' )
       _SCREAMING_SNAKE_CASE							    =kwargs.pop('generator' )
       _SCREAMING_SNAKE_CASE							    =decoder_config.pop('model_type' )

       from ..auto.configuration_auto import AutoConfig

       _SCREAMING_SNAKE_CASE							    =AutoConfig.for_model(_a							,					**_a )
       _SCREAMING_SNAKE_CASE							    =AutoConfig.for_model(_a							,					**_a )

       _SCREAMING_SNAKE_CASE							    =reduce_loss
       _SCREAMING_SNAKE_CASE							    =label_smoothing
       _SCREAMING_SNAKE_CASE							    =exclude_bos_score
       _SCREAMING_SNAKE_CASE							    =do_marginalize

       _SCREAMING_SNAKE_CASE							    =title_sep
       _SCREAMING_SNAKE_CASE							    =doc_sep
       _SCREAMING_SNAKE_CASE							    =n_docs
       _SCREAMING_SNAKE_CASE							    =max_combined_length

       _SCREAMING_SNAKE_CASE							    =dataset
       _SCREAMING_SNAKE_CASE							    =dataset_split
       _SCREAMING_SNAKE_CASE							    =index_name

       _SCREAMING_SNAKE_CASE							    =retrieval_vector_size
       _SCREAMING_SNAKE_CASE							    =retrieval_batch_size
       _SCREAMING_SNAKE_CASE							    =passages_path
       _SCREAMING_SNAKE_CASE							    =index_path
       _SCREAMING_SNAKE_CASE							    =use_dummy_dataset

       _SCREAMING_SNAKE_CASE							    =output_retrieved

       _SCREAMING_SNAKE_CASE							    =do_deduplication

       _SCREAMING_SNAKE_CASE							    =use_cache

       if self.forced_eos_token_id is None:
            _SCREAMING_SNAKE_CASE							    =getattr(self.generator							,					'forced_eos_token_id'							,					_a )





  @classmethod
  def 						A  (		cls      :	List[str]							,					_a      :	PretrainedConfig							,					_a      :	PretrainedConfig							,					**_a      :	Tuple )	->    PretrainedConfig:





       '''simple docstring'''






       return cls(question_encoder=question_encoder_config.to_dict()							,					generator=generator_config.to_dict()							,					**_a )





  def 						A  (		self      :	int )	->    List[str]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =copy.deepcopy(self.__dict__ )
       _SCREAMING_SNAKE_CASE							    =self.question_encoder.to_dict()
       _SCREAMING_SNAKE_CASE							    =self.generator.to_dict()
       _SCREAMING_SNAKE_CASE							    =self.__class__.model_type
       return output






from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy







@add_start_docstrings(
    "The RoBERTa Model transformer with early exiting (DeeRoBERTa). ",UpperCamelCase__,)
class 						lowercase							(   UpperCamelCase__		):
     _a							 =						RobertaConfig
     _a							 =						"roberta"

     def __init__(  self		,     _a				)						->							Optional[int]:
           super().__init__(_a				)

           _A							:     Union[str, Any]			=						RobertaEmbeddings(_a				)
           self.init_weights()







@add_start_docstrings(
    "RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,\n    also takes care of multi-layer training. ",UpperCamelCase__,)
class 						lowercase							(   UpperCamelCase__		):
     _a							 =						RobertaConfig
     _a							 =						"roberta"

     def __init__(  self		,     _a				)						->							str:
           super().__init__(_a				)
           _A							:     Any			=						config.num_labels
           _A							:     Dict			=						config.num_hidden_layers

           _A							:     List[str]			=						DeeRobertaModel(_a				)
           _A							:     int			=						nn.Dropout(config.hidden_dropout_prob				)
           _A							:     int			=						nn.Linear(config.hidden_size		,     self.config.num_labels				)



     @add_start_docstrings_to_model_forward(_a				)
     def 					a__	(  self		,     _a=None		,     _a=None		,     _a=None		,     _a=None		,     _a=None		,     _a=None		,     _a=None		,     _a=-1		,     _a=False		,     )						->							Any:
           _A							:     Optional[int]			=						self.num_layers
           try:
                 _A							:     List[str]			=						self.roberta(
                     _a		,     attention_mask=_a		,     token_type_ids=_a		,     position_ids=_a		,     head_mask=_a		,     inputs_embeds=_a		,     )

                 _A							:     List[str]			=						outputs[1]

                 _A							:     List[str]			=						self.dropout(_a				)
                 _A							:     Optional[Any]			=						self.classifier(_a				)
                 _A							:     List[Any]			=						(logits,) + outputs[2:]  # add hidden states and attention if they are here
           except HighwayException as e:
                 _A							:     List[Any]			=						e.message
                 _A							:     Optional[int]			=						e.exit_layer
                 _A							:     Optional[int]			=						outputs[0]

           if not self.training:
                 _A							:     int			=						entropy(_a				)
                 _A							:     int			=						[]
                 _A							:     int			=						[]
           if labels is not None:
                 if self.num_labels == 1:
                       #  We are doing regression
                       _A							:     Union[str, Any]			=						MSELoss()
                       _A							:     Tuple			=						loss_fct(logits.view(-1				)		,     labels.view(-1				)				)
                 else:
                       _A							:     List[Any]			=						CrossEntropyLoss()
                       _A							:     Dict			=						loss_fct(logits.view(-1		,     self.num_labels				)		,     labels.view(-1				)				)

                 # work with highway exits
                 _A							:     Optional[Any]			=						[]
                 for highway_exit in outputs[-1]:
                       _A							:     Tuple			=						highway_exit[0]
                       if not self.training:
                             highway_logits_all.append(_a				)
                             highway_entropy.append(highway_exit[2]				)
                       if self.num_labels == 1:
                             #  We are doing regression
                             _A							:     List[str]			=						MSELoss()
                             _A							:     Optional[int]			=						loss_fct(highway_logits.view(-1				)		,     labels.view(-1				)				)
                       else:
                             _A							:     List[Any]			=						CrossEntropyLoss()
                             _A							:     Tuple			=						loss_fct(highway_logits.view(-1		,     self.num_labels				)		,     labels.view(-1				)				)
                       highway_losses.append(_a				)

                 if train_highway:
                       _A							:     Dict			=						(sum(highway_losses[:-1]				),) + outputs
                       # exclude the final highway, of course
                 else:
                       _A							:     int			=						(loss,) + outputs
           if not self.training:
                 _A							:     Optional[Any]			=						outputs + ((original_entropy, highway_entropy), exit_layer)
                 if output_layer >= 0:
                       _A							:     Union[str, Any]			=						(
                           (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
                       )  # use the highway of the last layer

           return outputs  # (loss), logits, (hidden_states), (attentions), entropy

