




from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy







@add_start_docstrings(
    '''The RoBERTa Model transformer with early exiting (DeeRoBERTa). '''			,				__magic_name__			,				)
class     __lowerCAmelCase (       __magic_name__ ):

			"""simple docstring"""







			snake_case_   		=       RobertaConfig
			snake_case_   		=       '''roberta'''


			def __init__(      self			,						lowerCamelCase__							)    ->      str:



							'''simple docstring'''
							super().__init__(lowerCamelCase__							)

							__lowerCamelCase		       =    RobertaEmbeddings(lowerCamelCase__							)
							self.init_weights()







@add_start_docstrings(
    '''RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. '''			,				__magic_name__			,				)
class     __lowerCAmelCase (       __magic_name__ ):

			"""simple docstring"""







			snake_case_   		=       RobertaConfig
			snake_case_   		=       '''roberta'''


			def __init__(      self			,						lowerCamelCase__							)    ->      Optional[Any]:



							'''simple docstring'''
							super().__init__(lowerCamelCase__							)
							__lowerCamelCase		       =    config.num_labels
							__lowerCamelCase		       =    config.num_hidden_layers

							__lowerCamelCase		       =    DeeRobertaModel(lowerCamelCase__							)
							__lowerCamelCase		       =    nn.Dropout(config.hidden_dropout_prob							)
							__lowerCamelCase		       =    nn.Linear(config.hidden_size			,						self.config.num_labels							)






			@add_start_docstrings_to_model_forward(lowerCamelCase__							)
			def       lowercase_    (      self			,						lowerCamelCase__=None			,						lowerCamelCase__=None			,						lowerCamelCase__=None			,						lowerCamelCase__=None			,						lowerCamelCase__=None			,						lowerCamelCase__=None			,						lowerCamelCase__=None			,						lowerCamelCase__=-1			,						lowerCamelCase__=False			,						)    ->      str:



							'''simple docstring'''
							__lowerCamelCase		       =    self.num_layers
							try:
											__lowerCamelCase		       =    self.roberta(
											    lowerCamelCase__			,						attention_mask=lowerCamelCase__			,						token_type_ids=lowerCamelCase__			,						position_ids=lowerCamelCase__			,						head_mask=lowerCamelCase__			,						inputs_embeds=lowerCamelCase__			,						)

											__lowerCamelCase		       =    outputs[1]

											__lowerCamelCase		       =    self.dropout(lowerCamelCase__							)
											__lowerCamelCase		       =    self.classifier(lowerCamelCase__							)
											__lowerCamelCase		       =    (logits,) + outputs[2:]  # add hidden states and attention if they are here
							except HighwayException as e:
											__lowerCamelCase		       =    e.message
											__lowerCamelCase		       =    e.exit_layer
											__lowerCamelCase		       =    outputs[0]

							if not self.training:
											__lowerCamelCase		       =    entropy(lowerCamelCase__							)
											__lowerCamelCase		       =    []
											__lowerCamelCase		       =    []
							if labels is not None:
											if self.num_labels == 1:
															#  We are doing regression
															__lowerCamelCase		       =    MSELoss()
															__lowerCamelCase		       =    loss_fct(logits.view(-1							)			,						labels.view(-1							)							)
											else:
															__lowerCamelCase		       =    CrossEntropyLoss()
															__lowerCamelCase		       =    loss_fct(logits.view(-1			,						self.num_labels							)			,						labels.view(-1							)							)

											# work with highway exits
											__lowerCamelCase		       =    []
											for highway_exit in outputs[-1]:
															__lowerCamelCase		       =    highway_exit[0]
															if not self.training:
																			highway_logits_all.append(lowerCamelCase__							)
																			highway_entropy.append(highway_exit[2]							)
															if self.num_labels == 1:
																			#  We are doing regression
																			__lowerCamelCase		       =    MSELoss()
																			__lowerCamelCase		       =    loss_fct(highway_logits.view(-1							)			,						labels.view(-1							)							)
															else:
																			__lowerCamelCase		       =    CrossEntropyLoss()
																			__lowerCamelCase		       =    loss_fct(highway_logits.view(-1			,						self.num_labels							)			,						labels.view(-1							)							)
															highway_losses.append(lowerCamelCase__							)

											if train_highway:
															__lowerCamelCase		       =    (sum(highway_losses[:-1]							),) + outputs
															# exclude the final highway, of course
											else:
															__lowerCamelCase		       =    (loss,) + outputs
							if not self.training:
											__lowerCamelCase		       =    outputs + ((original_entropy, highway_entropy), exit_layer)
											if output_layer >= 0:
															__lowerCamelCase		       =    (
															    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
															)  # use the highway of the last layer

							return outputs  # (loss), logits, (hidden_states), (attentions), entropy



def lowerCamelCase  (   SCREAMING_SNAKE_CASE	):
    '''simple docstring'''







    __UpperCamelCase     :Tuple			  =		[0 for i in range(len(SCREAMING_SNAKE_CASE	)	)]

    # initialize interval's left pointer and right pointer
    __UpperCamelCase  ,       __UpperCamelCase     :str			  =		0, 0

    for i in range(1  ,			len(SCREAMING_SNAKE_CASE	)	):
        # case when current index is inside the interval
        if i <= right_pointer:
            __UpperCamelCase     :Union[str, Any]			  =		min(right_pointer - i + 1  ,			z_result[i - left_pointer]	)
            __UpperCamelCase     :Tuple			  =		min_edge

        while go_next(SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE	):
            z_result[i] += 1

        # if new index's result gives us more right interval,
        # we've to update left_pointer and right_pointer
        if i + z_result[i] - 1 > right_pointer:
            __UpperCamelCase  ,       __UpperCamelCase     :Union[str, Any]			  =		i, i + z_result[i] - 1

    return z_result



def lowerCamelCase  (   SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE	):
    '''simple docstring'''







    return i + z_result[i] < len(SCREAMING_SNAKE_CASE	) and s[z_result[i]] == s[i + z_result[i]]



def lowerCamelCase  (   SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE	):
    '''simple docstring'''







    __UpperCamelCase     :List[Any]			  =		0
    # concatenate 'pattern' and 'input_str' and call z_function
    # with concatenated string
    __UpperCamelCase     :Tuple			  =		z_function(pattern + input_str	)

    for val in z_result:
        # if value is greater then length of the pattern string
        # that means this index is starting position of substring
        # which is equal to pattern string
        if val >= len(SCREAMING_SNAKE_CASE	):
            answer += 1

    return answer


if __name__ == "__main__":
    import doctest

    doctest.testmod()





