




import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


_snake_case										=       logging.getLogger(__name__)







class 						lowercase							:

     def __init__(  self				)						->							Union[str, Any]:
           _A							:     Tuple			=						False

     def 					a__	(  self		,     _a		,     _a		,     _a		,     _a				)						->							Optional[int]:
           if not self.initialized:
                 _A							:     Any			=						RagRetriever(
                     _a		,     question_encoder_tokenizer=_a		,     generator_tokenizer=_a		,     index=_a		,     init_retrieval=_a		,     )
                 _A							:     Any			=						True

     def 					a__	(  self				)						->							Dict:
           self.retriever.index.init_index()



     def 					a__	(  self		,     _a		,     _a				)						->							Any:
           _A			,   _A							:     List[str]			=						self.retriever._main_retrieve(_a		,     _a				)
           return doc_ids, retrieved_doc_embeds







class 						lowercase							(   UpperCamelCase__		):

     def __init__(  self		,     _a		,     _a		,     _a		,     _a		,     _a=None				)						->							Dict:
           if index is not None and index.is_initialized() and len(_a				) > 0:
                 raise ValueError(
                     """When using Ray for distributed fine-tuning, """
                     """you'll need to provide the paths instead, """
                     """as the dataset and the index are loaded """
                     """separately. More info in examples/rag/use_own_knowledge_dataset.py """				)
           super().__init__(
               _a		,     question_encoder_tokenizer=_a		,     generator_tokenizer=_a		,     index=_a		,     init_retrieval=_a		,     )
           _A							:     int			=						retrieval_workers
           if len(self.retrieval_workers				) > 0:
                 ray.get(
                     [
                         worker.create_rag_retriever.remote(_a		,     _a		,     _a		,     _a				)
                         for worker in self.retrieval_workers
                     ]				)

     def 					a__	(  self				)						->							Union[str, Any]:
           logger.info("""initializing retrieval"""				)

           if len(self.retrieval_workers				) > 0:
                 ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]				)
           else:
                 # Non-distributed training. Load index into this same process.
                 self.index.init_index()

     def 					a__	(  self		,     _a		,     _a				)						->							List[str]:
           if len(self.retrieval_workers				) > 0:
                 # Select a random retrieval actor.
                 _A							:     Optional[Any]			=						self.retrieval_workers[random.randint(0		,     len(self.retrieval_workers				) - 1				)]
                 _A			,   _A							:     List[str]			=						ray.get(random_worker.retrieve.remote(_a		,     _a				)				)
           else:
                 _A			,   _A							:     Optional[int]			=						self._main_retrieve(_a		,     _a				)
           return retrieved_doc_embeds, doc_ids, self.index.get_doc_dicts(_a				)

     @classmethod
     def 					a__	(  cls		,     _a		,     _a=None		,     **_a				)						->							List[Any]:
           return super(_a		,     cls				).get_tokenizers(_a		,     _a		,     **_a				)



     @classmethod
     def 					a__	(  cls		,     _a		,     _a		,     _a=None		,     **_a				)						->							Union[str, Any]:
           _A							:     Any			=						kwargs.pop("""config"""		,     _a				) or RagConfig.from_pretrained(_a		,     **_a				)
           _A							:     str			=						RagTokenizer.from_pretrained(_a		,     config=_a				)
           _A							:     Dict			=						rag_tokenizer.question_encoder
           _A							:     Any			=						rag_tokenizer.generator
           if indexed_dataset is not None:
                 _A							:     Optional[int]			=						"""custom"""
                 _A							:     Any			=						CustomHFIndex(config.retrieval_vector_size		,     _a				)
           else:
                 _A							:     int			=						cls._build_index(_a				)
           return cls(
               _a		,     question_encoder_tokenizer=_a		,     generator_tokenizer=_a		,     retrieval_workers=_a		,     index=_a		,     )







from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy







@add_start_docstrings(
    "The RoBERTa Model transformer with early exiting (DeeRoBERTa). ",UpperCamelCase__,)
class 						lowercase							(   UpperCamelCase__		):
     _a							 =						RobertaConfig
     _a							 =						"roberta"

     def __init__(  self		,     _a				)						->							Optional[int]:
           super().__init__(_a				)

           _A							:     Union[str, Any]			=						RobertaEmbeddings(_a				)
           self.init_weights()







@add_start_docstrings(
    "RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,\n    also takes care of multi-layer training. ",UpperCamelCase__,)
class 						lowercase							(   UpperCamelCase__		):
     _a							 =						RobertaConfig
     _a							 =						"roberta"

     def __init__(  self		,     _a				)						->							str:
           super().__init__(_a				)
           _A							:     Any			=						config.num_labels
           _A							:     Dict			=						config.num_hidden_layers

           _A							:     List[str]			=						DeeRobertaModel(_a				)
           _A							:     int			=						nn.Dropout(config.hidden_dropout_prob				)
           _A							:     int			=						nn.Linear(config.hidden_size		,     self.config.num_labels				)



     @add_start_docstrings_to_model_forward(_a				)
     def 					a__	(  self		,     _a=None		,     _a=None		,     _a=None		,     _a=None		,     _a=None		,     _a=None		,     _a=None		,     _a=-1		,     _a=False		,     )						->							Any:
           _A							:     Optional[int]			=						self.num_layers
           try:
                 _A							:     List[str]			=						self.roberta(
                     _a		,     attention_mask=_a		,     token_type_ids=_a		,     position_ids=_a		,     head_mask=_a		,     inputs_embeds=_a		,     )

                 _A							:     List[str]			=						outputs[1]

                 _A							:     List[str]			=						self.dropout(_a				)
                 _A							:     Optional[Any]			=						self.classifier(_a				)
                 _A							:     List[Any]			=						(logits,) + outputs[2:]  # add hidden states and attention if they are here
           except HighwayException as e:
                 _A							:     List[Any]			=						e.message
                 _A							:     Optional[int]			=						e.exit_layer
                 _A							:     Optional[int]			=						outputs[0]

           if not self.training:
                 _A							:     int			=						entropy(_a				)
                 _A							:     int			=						[]
                 _A							:     int			=						[]
           if labels is not None:
                 if self.num_labels == 1:
                       #  We are doing regression
                       _A							:     Union[str, Any]			=						MSELoss()
                       _A							:     Tuple			=						loss_fct(logits.view(-1				)		,     labels.view(-1				)				)
                 else:
                       _A							:     List[Any]			=						CrossEntropyLoss()
                       _A							:     Dict			=						loss_fct(logits.view(-1		,     self.num_labels				)		,     labels.view(-1				)				)

                 # work with highway exits
                 _A							:     Optional[Any]			=						[]
                 for highway_exit in outputs[-1]:
                       _A							:     Tuple			=						highway_exit[0]
                       if not self.training:
                             highway_logits_all.append(_a				)
                             highway_entropy.append(highway_exit[2]				)
                       if self.num_labels == 1:
                             #  We are doing regression
                             _A							:     List[str]			=						MSELoss()
                             _A							:     Optional[int]			=						loss_fct(highway_logits.view(-1				)		,     labels.view(-1				)				)
                       else:
                             _A							:     List[Any]			=						CrossEntropyLoss()
                             _A							:     Tuple			=						loss_fct(highway_logits.view(-1		,     self.num_labels				)		,     labels.view(-1				)				)
                       highway_losses.append(_a				)

                 if train_highway:
                       _A							:     Dict			=						(sum(highway_losses[:-1]				),) + outputs
                       # exclude the final highway, of course
                 else:
                       _A							:     int			=						(loss,) + outputs
           if not self.training:
                 _A							:     Optional[Any]			=						outputs + ((original_entropy, highway_entropy), exit_layer)
                 if output_layer >= 0:
                       _A							:     Union[str, Any]			=						(
                           (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
                       )  # use the highway of the last layer

           return outputs  # (loss), logits, (hidden_states), (attentions), entropy

