


from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy



@add_start_docstrings(
    """The RoBERTa Model transformer with early exiting (DeeRoBERTa). """							,	_a							,	)
class      __lowerCamelCase       (_a		):
       _lowercase											=			RobertaConfig
       _lowercase											=			'''roberta'''







       def __init__(       self:      Dict,A_:      Union[str, Any] ):






            '''simple docstring'''




            super().__init__(_snake_case )

            __UpperCamelCase      =       RobertaEmbeddings(_snake_case )
            self.init_weights()



@add_start_docstrings(
    """RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. """							,	_a							,	)
class      __lowerCamelCase       (_a		):
       _lowercase											=			RobertaConfig
       _lowercase											=			'''roberta'''







       def __init__(       self:      int,A_:      Tuple ):






            '''simple docstring'''




            super().__init__(_snake_case )
            __UpperCamelCase      =       config.num_labels
            __UpperCamelCase      =       config.num_hidden_layers

            __UpperCamelCase      =       DeeRobertaModel(_snake_case )
            __UpperCamelCase      =       nn.Dropout(config.hidden_dropout_prob )
            __UpperCamelCase      =       nn.Linear(config.hidden_size,self.config.num_labels )







       @add_start_docstrings_to_model_forward(_snake_case )
       def 						snake_case_     (       self:      Union[str, Any],A_:      Union[str, Any]=None,A_:      Optional[int]=None,A_:      Tuple=None,A_:      Any=None,A_:      List[str]=None,A_:      int=None,A_:      int=None,A_:      Union[str, Any]=-1,A_:      List[str]=False,):






            '''simple docstring'''




            __UpperCamelCase      =       self.num_layers
            try:
                 __UpperCamelCase      =       self.roberta(
                     _snake_case,attention_mask=_snake_case,token_type_ids=_snake_case,position_ids=_snake_case,head_mask=_snake_case,inputs_embeds=_snake_case,)

                 __UpperCamelCase      =       outputs[1]

                 __UpperCamelCase      =       self.dropout(_snake_case )
                 __UpperCamelCase      =       self.classifier(_snake_case )
                 __UpperCamelCase      =       (logits,) + outputs[2:]  # add hidden states and attention if they are here
            except HighwayException as e:
                 __UpperCamelCase      =       e.message
                 __UpperCamelCase      =       e.exit_layer
                 __UpperCamelCase      =       outputs[0]

            if not self.training:
                 __UpperCamelCase      =       entropy(_snake_case )
                 __UpperCamelCase      =       []
                 __UpperCamelCase      =       []
            if labels is not None:
                 if self.num_labels == 1:
                      #  We are doing regression
                      __UpperCamelCase      =       MSELoss()
                      __UpperCamelCase      =       loss_fct(logits.view(-1 ),labels.view(-1 ) )
                 else:
                      __UpperCamelCase      =       CrossEntropyLoss()
                      __UpperCamelCase      =       loss_fct(logits.view(-1,self.num_labels ),labels.view(-1 ) )

                 # work with highway exits
                 __UpperCamelCase      =       []
                 for highway_exit in outputs[-1]:
                      __UpperCamelCase      =       highway_exit[0]
                      if not self.training:
                           highway_logits_all.append(_snake_case )
                           highway_entropy.append(highway_exit[2] )
                      if self.num_labels == 1:
                           #  We are doing regression
                           __UpperCamelCase      =       MSELoss()
                           __UpperCamelCase      =       loss_fct(highway_logits.view(-1 ),labels.view(-1 ) )
                      else:
                           __UpperCamelCase      =       CrossEntropyLoss()
                           __UpperCamelCase      =       loss_fct(highway_logits.view(-1,self.num_labels ),labels.view(-1 ) )
                      highway_losses.append(_snake_case )

                 if train_highway:
                      __UpperCamelCase      =       (sum(highway_losses[:-1] ),) + outputs
                      # exclude the final highway, of course
                 else:
                      __UpperCamelCase      =       (loss,) + outputs
            if not self.training:
                 __UpperCamelCase      =       outputs + ((original_entropy, highway_entropy), exit_layer)
                 if output_layer >= 0:
                      __UpperCamelCase      =       (
                          (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
                      )  # use the highway of the last layer

            return outputs  # (loss), logits, (hidden_states), (attentions), entropy








from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case_ : int    		=       logging.get_logger(__name__)

snake_case_ : str    		=       {}







class __snake_case			(			a		):
   UpperCAmelCase__  :    str          =  '''llama'''
   UpperCAmelCase__  :    Dict          =  ['''past_key_values''']







   def __init__(      self		:						str					,						_snake_case		:						List[str]=32000					,						_snake_case		:						int=4096					,						_snake_case		:						List[str]=11008					,						_snake_case		:						Optional[int]=32					,						_snake_case		:						List[Any]=32					,						_snake_case		:						Tuple=None					,						_snake_case		:						int="silu"					,						_snake_case		:						List[Any]=2048					,						_snake_case		:						List[str]=0.0_2					,						_snake_case		:						Any=1e-6					,						_snake_case		:						List[str]=True					,						_snake_case		:						Optional[Any]=0					,						_snake_case		:						Dict=1					,						_snake_case		:						List[Any]=2					,						_snake_case		:						str=1					,						_snake_case		:						Union[str, Any]=False					,						_snake_case		:						str=None					,						**_snake_case		:						List[Any]					,						):

        """simple docstring"""


        UpperCAmelCase_          =							vocab_size
        UpperCAmelCase_          =							max_position_embeddings
        UpperCAmelCase_          =							hidden_size
        UpperCAmelCase_          =							intermediate_size
        UpperCAmelCase_          =							num_hidden_layers
        UpperCAmelCase_          =							num_attention_heads

        # for backward compatibility
        if num_key_value_heads is None:
             UpperCAmelCase_          =							num_attention_heads

        UpperCAmelCase_          =							num_key_value_heads
        UpperCAmelCase_          =							hidden_act
        UpperCAmelCase_          =							initializer_range
        UpperCAmelCase_          =							rms_norm_eps
        UpperCAmelCase_          =							pretraining_tp
        UpperCAmelCase_          =							use_cache
        UpperCAmelCase_          =							rope_scaling
        self._rope_scaling_validation()

        super().__init__(
            pad_token_id=_snake_case					,						bos_token_id=_snake_case					,						eos_token_id=_snake_case					,						tie_word_embeddings=_snake_case					,						**_snake_case					,						)







   def lowerCamelCase						(      self		:						Optional[int]):

        """simple docstring"""


        if self.rope_scaling is None:
             return

        if not isinstance(self.rope_scaling					,						_snake_case) or len(self.rope_scaling) != 2:
             raise ValueError(
                 '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
                 F"""got {self.rope_scaling}""")
        UpperCAmelCase_          =							self.rope_scaling.get('''type'''					,						_snake_case)
        UpperCAmelCase_          =							self.rope_scaling.get('''factor'''					,						_snake_case)
        if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
             raise ValueError(
                 F"""`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}""")
        if rope_scaling_factor is None or not isinstance(_snake_case					,						_snake_case) or rope_scaling_factor <= 1.0:
             raise ValueError(F"""`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}""")






