






import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


_a      	=  logging.getLogger(__name__)





class __lowerCamelCase (      snake_case__):





	"""simple docstring"""






	def        UpperCamelCase    (							self       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase=None       ,    UpperCAmelCase=None    ):
							"""simple docstring"""







							_UpperCAmelCase									=			self.layer[current_layer](UpperCAmelCase       ,    UpperCAmelCase       ,    head_mask[current_layer]    )

							_UpperCAmelCase									=			layer_outputs[0]

							return hidden_states





@add_start_docstrings(
    "The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top."     ,       snake_case__     ,       )
class __lowerCamelCase (      snake_case__):





	"""simple docstring"""






	def __init__(							self       ,    UpperCAmelCase    ):
							"""simple docstring"""







							super().__init__(UpperCAmelCase    )

							_UpperCAmelCase									=			BertEncoderWithPabee(UpperCAmelCase    )

							self.init_weights()
							_UpperCAmelCase									=			0
							_UpperCAmelCase									=			0
							_UpperCAmelCase									=			0

							_UpperCAmelCase									=			0

	def        UpperCamelCase    (							self       ,    UpperCAmelCase    ):
							"""simple docstring"""







							_UpperCAmelCase									=			threshold

	def        UpperCamelCase    (							self       ,    UpperCAmelCase    ):
							"""simple docstring"""







							_UpperCAmelCase									=			patience

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			0
							_UpperCAmelCase									=			0

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			self.inference_layers_num / self.inference_instances_num
							_UpperCAmelCase									=			(
							    F"""*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="""
							    F""" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"""
							)
							print(UpperCAmelCase    )

	@add_start_docstrings_to_model_forward(UpperCAmelCase    )
	def        UpperCamelCase    (							self       ,    UpperCAmelCase=None       ,    UpperCAmelCase=None       ,    UpperCAmelCase=None       ,    UpperCAmelCase=None       ,    UpperCAmelCase=None       ,    UpperCAmelCase=None       ,    UpperCAmelCase=None       ,    UpperCAmelCase=None       ,    UpperCAmelCase=None       ,    UpperCAmelCase=None       ,    UpperCAmelCase=False       ,    ):
							"""simple docstring"""







							if input_ids is not None and inputs_embeds is not None:
													raise ValueError('You cannot specify both input_ids and inputs_embeds at the same time'    )
							elif input_ids is not None:
													_UpperCAmelCase									=			input_ids.size()
							elif inputs_embeds is not None:
													_UpperCAmelCase									=			inputs_embeds.size()[:-1]
							else:
													raise ValueError('You have to specify either input_ids or inputs_embeds'    )

							_UpperCAmelCase									=			input_ids.device if input_ids is not None else inputs_embeds.device

							if attention_mask is None:
													_UpperCAmelCase									=			torch.ones(UpperCAmelCase       ,    device=UpperCAmelCase    )
							if token_type_ids is None:
													_UpperCAmelCase									=			torch.zeros(UpperCAmelCase       ,    dtype=torch.long       ,    device=UpperCAmelCase    )

							# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
							# ourselves in which case we just need to make it broadcastable to all heads.
							_UpperCAmelCase									=			self.get_extended_attention_mask(UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase    )

							# If a 2D ou 3D attention mask is provided for the cross-attention
							# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
							if self.config.is_decoder and encoder_hidden_states is not None:
													_UpperCAmelCase					, _UpperCAmelCase					, _UpperCAmelCase									=			encoder_hidden_states.size()
													_UpperCAmelCase									=			(encoder_batch_size, encoder_sequence_length)
													if encoder_attention_mask is None:
																			_UpperCAmelCase									=			torch.ones(UpperCAmelCase       ,    device=UpperCAmelCase    )
													_UpperCAmelCase									=			self.invert_attention_mask(UpperCAmelCase    )
							else:
													_UpperCAmelCase									=			None

							# Prepare head mask if needed
							# 1.0 in head_mask indicate we keep the head
							# attention_probs has shape bsz x n_heads x N x N
							# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
							# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
							_UpperCAmelCase									=			self.get_head_mask(UpperCAmelCase       ,    self.config.num_hidden_layers    )

							_UpperCAmelCase									=			self.embeddings(
							    input_ids=UpperCAmelCase       ,    position_ids=UpperCAmelCase       ,    token_type_ids=UpperCAmelCase       ,    inputs_embeds=UpperCAmelCase    )
							_UpperCAmelCase									=			embedding_output

							if self.training:
													_UpperCAmelCase									=			[]
													for i in range(self.config.num_hidden_layers    ):
																			_UpperCAmelCase									=			self.encoder.adaptive_forward(
																			    UpperCAmelCase       ,    current_layer=UpperCAmelCase       ,    attention_mask=UpperCAmelCase       ,    head_mask=UpperCAmelCase    )

																			_UpperCAmelCase									=			self.pooler(UpperCAmelCase    )
																			_UpperCAmelCase									=			output_layers[i](output_dropout(UpperCAmelCase    )    )
																			res.append(UpperCAmelCase    )
							elif self.patience == 0:  # Use all layers for inference
													_UpperCAmelCase									=			self.encoder(
													    UpperCAmelCase       ,    attention_mask=UpperCAmelCase       ,    head_mask=UpperCAmelCase       ,    encoder_hidden_states=UpperCAmelCase       ,    encoder_attention_mask=UpperCAmelCase       ,    )
													_UpperCAmelCase									=			self.pooler(encoder_outputs[0]    )
													_UpperCAmelCase									=			[output_layers[self.config.num_hidden_layers - 1](UpperCAmelCase    )]
							else:
													_UpperCAmelCase									=			0
													_UpperCAmelCase									=			None
													_UpperCAmelCase									=			0
													for i in range(self.config.num_hidden_layers    ):
																			calculated_layer_num += 1
																			_UpperCAmelCase									=			self.encoder.adaptive_forward(
																			    UpperCAmelCase       ,    current_layer=UpperCAmelCase       ,    attention_mask=UpperCAmelCase       ,    head_mask=UpperCAmelCase    )

																			_UpperCAmelCase									=			self.pooler(UpperCAmelCase    )
																			_UpperCAmelCase									=			output_layers[i](UpperCAmelCase    )
																			if regression:
																									_UpperCAmelCase									=			logits.detach()
																									if patient_result is not None:
																															_UpperCAmelCase									=			patient_result.detach()
																									if (patient_result is not None) and torch.abs(patient_result - labels    ) < self.regression_threshold:
																															patient_counter += 1
																									else:
																															_UpperCAmelCase									=			0
																			else:
																									_UpperCAmelCase									=			logits.detach().argmax(dim=1    )
																									if patient_result is not None:
																															_UpperCAmelCase									=			patient_result.detach().argmax(dim=1    )
																									if (patient_result is not None) and torch.all(labels.eq(UpperCAmelCase    )    ):
																															patient_counter += 1
																									else:
																															_UpperCAmelCase									=			0

																			_UpperCAmelCase									=			logits
																			if patient_counter == self.patience:
																									break
													_UpperCAmelCase									=			[patient_result]
													self.inference_layers_num += calculated_layer_num
													self.inference_instances_num += 1

							return res





@add_start_docstrings(
    "Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. "     ,       snake_case__     ,       )
class __lowerCamelCase (      snake_case__):





	"""simple docstring"""






	def __init__(							self       ,    UpperCAmelCase    ):
							"""simple docstring"""







							super().__init__(UpperCAmelCase    )
							_UpperCAmelCase									=			config.num_labels

							_UpperCAmelCase									=			BertModelWithPabee(UpperCAmelCase    )
							_UpperCAmelCase									=			nn.Dropout(config.hidden_dropout_prob    )
							_UpperCAmelCase									=			nn.ModuleList(
							    [nn.Linear(config.hidden_size       ,    self.config.num_labels    ) for _ in range(config.num_hidden_layers    )]    )

							self.init_weights()

	@add_start_docstrings_to_model_forward(UpperCAmelCase    )
	def        UpperCamelCase    (							self       ,    UpperCAmelCase=None       ,    UpperCAmelCase=None       ,    UpperCAmelCase=None       ,    UpperCAmelCase=None       ,    UpperCAmelCase=None       ,    UpperCAmelCase=None       ,    UpperCAmelCase=None       ,    ):
							"""simple docstring"""







							_UpperCAmelCase									=			self.bert(
							    input_ids=UpperCAmelCase       ,    attention_mask=UpperCAmelCase       ,    token_type_ids=UpperCAmelCase       ,    position_ids=UpperCAmelCase       ,    head_mask=UpperCAmelCase       ,    inputs_embeds=UpperCAmelCase       ,    output_dropout=self.dropout       ,    output_layers=self.classifiers       ,    regression=self.num_labels == 1       ,    )

							_UpperCAmelCase									=			(logits[-1],)

							if labels is not None:
													_UpperCAmelCase									=			None
													_UpperCAmelCase									=			0
													for ix, logits_item in enumerate(UpperCAmelCase    ):
																			if self.num_labels == 1:
																									#  We are doing regression
																									_UpperCAmelCase									=			MSELoss()
																									_UpperCAmelCase									=			loss_fct(logits_item.view(-1    )       ,    labels.view(-1    )    )
																			else:
																									_UpperCAmelCase									=			CrossEntropyLoss()
																									_UpperCAmelCase									=			loss_fct(logits_item.view(-1       ,    self.num_labels    )       ,    labels.view(-1    )    )
																			if total_loss is None:
																									_UpperCAmelCase									=			loss
																			else:
																									total_loss += loss * (ix + 1)
																			total_weights += ix + 1
													_UpperCAmelCase									=			(total_loss / total_weights,) + outputs

							return outputs


SCREAMING_SNAKE_CASE							:Any							 =				256
# Modulus to hash a string
SCREAMING_SNAKE_CASE							:Union[str, Any]							 =				100_0003


def 						UpperCAmelCase			(	a_		,   a_						)		->	bool:







				"""simple docstring"""



				__A   			= len(a_						)
				__A   			= len(a_						)
				if p_len > t_len:
								return False

				__A   			= 0
				__A   			= 0
				__A   			= 1

				# Calculating the hash of pattern and substring of text
				for i in range(a_						):
								__A   			= (ord(pattern[i]						) + p_hash * alphabet_size) % modulus
								__A   			= (ord(text[i]						) + text_hash * alphabet_size) % modulus
								if i == p_len - 1:
												continue
								__A   			= (modulus_power * alphabet_size) % modulus

				for i in range(0		,   t_len - p_len + 1						):
								if text_hash == p_hash and text[i : i + p_len] == pattern:
												return True
								if i == t_len - p_len:
												continue
								# Calculate the https://en.wikipedia.org/wiki/Rolling_hash
								__A   			= (
								    (text_hash - ord(text[i]						) * modulus_power) * alphabet_size
								    + ord(text[i + p_len]						)
								) % modulus
				return False


def 						UpperCAmelCase			(	)		->	None:







				"""simple docstring"""



				__A   			= "abc1abc12"
				__A   			= "alskfjaldsabc1abc1abc12k23adsfabcabc"
				__A   			= "alskfjaldsk23adsfabcabc"
				assert rabin_karp(a_		,   a_						) and not rabin_karp(a_		,   a_						)

				# Test 2)
				__A   			= "ABABX"
				__A   			= "ABABZABABYABABX"
				assert rabin_karp(a_		,   a_						)

				# Test 3)
				__A   			= "AAAB"
				__A   			= "ABAAAAAB"
				assert rabin_karp(a_		,   a_						)

				# Test 4)
				__A   			= "abcdabcy"
				__A   			= "abcxabcdabxabcdabcdabcy"
				assert rabin_karp(a_		,   a_						)

				# Test 5)
				__A   			= "Lü"
				__A   			= "Lüsai"
				assert rabin_karp(a_		,   a_						)
				__A   			= "Lue"
				assert not rabin_karp(a_		,   a_						)
				print("Success."						)


if __name__ == "__main__":
					test_rabin_karp()
