


import torch
from torch import nn







class  snake_case__(							nn.Module							):







						'''simple docstring'''




						def __init__( self      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase=1      ,  __lowercase=False							)    ->			Any:
										super().__init__()

										lowerCAmelCase_       :					str			 = n_token
										lowerCAmelCase_       :					List[str]			 = d_embed
										lowerCAmelCase_       :					Optional[int]			 = d_proj

										lowerCAmelCase_       :					Dict			 = cutoffs + [n_token]
										lowerCAmelCase_       :					Dict			 = [0] + self.cutoffs
										lowerCAmelCase_       :					Union[str, Any]			 = div_val

										lowerCAmelCase_       :					Tuple			 = self.cutoffs[0]
										lowerCAmelCase_       :					Optional[int]			 = len(self.cutoffs							) - 1
										lowerCAmelCase_       :					Dict			 = self.shortlist_size + self.n_clusters

										if self.n_clusters > 0:
														lowerCAmelCase_       :					Union[str, Any]			 = nn.Parameter(torch.zeros(self.n_clusters      ,  self.d_embed							)							)
														lowerCAmelCase_       :					Dict			 = nn.Parameter(torch.zeros(self.n_clusters							)							)

										lowerCAmelCase_       :					Dict			 = nn.ModuleList()
										lowerCAmelCase_       :					Optional[int]			 = nn.ParameterList()

										if div_val == 1:
														for i in range(len(self.cutoffs							)							):
																		if d_proj != d_embed:
																						self.out_projs.append(nn.Parameter(torch.FloatTensor(SCREAMING_SNAKE_CASE_      ,  SCREAMING_SNAKE_CASE_							)							)							)
																		else:
																						self.out_projs.append(SCREAMING_SNAKE_CASE_							)

														self.out_layers.append(nn.Linear(SCREAMING_SNAKE_CASE_      ,  SCREAMING_SNAKE_CASE_							)							)
										else:
														for i in range(len(self.cutoffs							)							):
																		lowerCAmelCase_ ,     lowerCAmelCase_       :					List[str]			 = self.cutoff_ends[i], self.cutoff_ends[i + 1]
																		lowerCAmelCase_       :					Union[str, Any]			 = d_embed // (div_val**i)

																		self.out_projs.append(nn.Parameter(torch.FloatTensor(SCREAMING_SNAKE_CASE_      ,  SCREAMING_SNAKE_CASE_							)							)							)

																		self.out_layers.append(nn.Linear(SCREAMING_SNAKE_CASE_      ,  r_idx - l_idx							)							)

										lowerCAmelCase_       :					int			 = keep_order




						def   lowercase_       ( self      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase							)    ->			Tuple:
										if proj is None:
														lowerCAmelCase_       :					int			 = nn.functional.linear(SCREAMING_SNAKE_CASE_      ,  SCREAMING_SNAKE_CASE_      ,  bias=SCREAMING_SNAKE_CASE_							)
										else:
														# if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
														lowerCAmelCase_       :					Tuple			 = nn.functional.linear(SCREAMING_SNAKE_CASE_      ,  proj.t().contiguous()							)
														lowerCAmelCase_       :					List[Any]			 = nn.functional.linear(SCREAMING_SNAKE_CASE_      ,  SCREAMING_SNAKE_CASE_      ,  bias=SCREAMING_SNAKE_CASE_							)
														# else:
														#     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
														#     if bias is not None:
														#         logit = logit + bias

										return logit




						def   lowercase_       ( self      ,  __lowercase      ,  __lowercase=None      ,  __lowercase=False							)    ->			Optional[Any]:

										if labels is not None:
														# Shift so that tokens < n predict n
														lowerCAmelCase_       :					Dict			 = hidden[..., :-1, :].contiguous()
														lowerCAmelCase_       :					Union[str, Any]			 = labels[..., 1:].contiguous()
														lowerCAmelCase_       :					List[str]			 = hidden.view(-1      ,  hidden.size(-1							)							)
														lowerCAmelCase_       :					Union[str, Any]			 = labels.view(-1							)
														if hidden.size(0							) != labels.size(0							):
																		raise RuntimeError('''Input and labels should have the same size in the batch dimension.'''							)
										else:
														lowerCAmelCase_       :					Dict			 = hidden.view(-1      ,  hidden.size(-1							)							)

										if self.n_clusters == 0:
														lowerCAmelCase_       :					List[Any]			 = self._compute_logit(SCREAMING_SNAKE_CASE_      ,  self.out_layers[0].weight      ,  self.out_layers[0].bias      ,  self.out_projs[0]							)
														if labels is not None:
																		lowerCAmelCase_       :					Tuple			 = labels != -1_0_0
																		lowerCAmelCase_       :					Union[str, Any]			 = torch.zeros_like(SCREAMING_SNAKE_CASE_      ,  dtype=hidden.dtype      ,  device=hidden.device							)
																		lowerCAmelCase_       :					Dict			 = (
																		    -nn.functional.log_softmax(SCREAMING_SNAKE_CASE_      ,  dim=-1							)[mask].gather(1      ,  labels[mask].unsqueeze(1							)							).squeeze(1							)
																		)
														else:
																		lowerCAmelCase_       :					Optional[int]			 = nn.functional.log_softmax(SCREAMING_SNAKE_CASE_      ,  dim=-1							)
										else:
														# construct weights and biases
														lowerCAmelCase_ ,     lowerCAmelCase_       :					List[str]			 = [], []
														for i in range(len(self.cutoffs							)							):
																		if self.div_val == 1:
																						lowerCAmelCase_ ,     lowerCAmelCase_       :					Dict			 = self.cutoff_ends[i], self.cutoff_ends[i + 1]
																						lowerCAmelCase_       :					Optional[Any]			 = self.out_layers[0].weight[l_idx:r_idx]
																						lowerCAmelCase_       :					Tuple			 = self.out_layers[0].bias[l_idx:r_idx]
																		else:
																						lowerCAmelCase_       :					List[Any]			 = self.out_layers[i].weight
																						lowerCAmelCase_       :					Union[str, Any]			 = self.out_layers[i].bias

																		if i == 0:
																						lowerCAmelCase_       :					int			 = torch.cat([weight_i, self.cluster_weight]      ,  dim=0							)
																						lowerCAmelCase_       :					Dict			 = torch.cat([bias_i, self.cluster_bias]      ,  dim=0							)

																		weights.append(SCREAMING_SNAKE_CASE_							)
																		biases.append(SCREAMING_SNAKE_CASE_							)

														lowerCAmelCase_ ,     lowerCAmelCase_ ,     lowerCAmelCase_       :					Union[str, Any]			 = weights[0], biases[0], self.out_projs[0]

														lowerCAmelCase_       :					List[str]			 = self._compute_logit(SCREAMING_SNAKE_CASE_      ,  SCREAMING_SNAKE_CASE_      ,  SCREAMING_SNAKE_CASE_      ,  SCREAMING_SNAKE_CASE_							)
														lowerCAmelCase_       :					int			 = nn.functional.log_softmax(SCREAMING_SNAKE_CASE_      ,  dim=1							)

														if labels is None:
																		lowerCAmelCase_       :					Optional[int]			 = hidden.new_empty((head_logit.size(0							), self.n_token)							)
														else:
																		lowerCAmelCase_       :					str			 = torch.zeros_like(SCREAMING_SNAKE_CASE_      ,  dtype=hidden.dtype      ,  device=hidden.device							)

														lowerCAmelCase_       :					Union[str, Any]			 = 0
														lowerCAmelCase_       :					Dict			 = [0] + self.cutoffs
														for i in range(len(SCREAMING_SNAKE_CASE_							) - 1							):
																		lowerCAmelCase_ ,     lowerCAmelCase_       :					Tuple			 = cutoff_values[i], cutoff_values[i + 1]

																		if labels is not None:
																						lowerCAmelCase_       :					Dict			 = (labels >= l_idx) & (labels < r_idx)
																						lowerCAmelCase_       :					str			 = mask_i.nonzero().squeeze()

																						if indices_i.numel() == 0:
																										continue

																						lowerCAmelCase_       :					List[str]			 = labels.index_select(0      ,  SCREAMING_SNAKE_CASE_							) - l_idx
																						lowerCAmelCase_       :					Optional[int]			 = head_logprob.index_select(0      ,  SCREAMING_SNAKE_CASE_							)
																						lowerCAmelCase_       :					Union[str, Any]			 = hidden.index_select(0      ,  SCREAMING_SNAKE_CASE_							)
																		else:
																						lowerCAmelCase_       :					str			 = hidden

																		if i == 0:
																						if labels is not None:
																										lowerCAmelCase_       :					Optional[Any]			 = head_logprob_i.gather(1      ,  target_i[:, None]							).squeeze(1							)
																						else:
																										lowerCAmelCase_       :					Dict			 = head_logprob[:, : self.cutoffs[0]]
																		else:
																						lowerCAmelCase_ ,     lowerCAmelCase_ ,     lowerCAmelCase_       :					Tuple			 = weights[i], biases[i], self.out_projs[i]

																						lowerCAmelCase_       :					Union[str, Any]			 = self._compute_logit(SCREAMING_SNAKE_CASE_      ,  SCREAMING_SNAKE_CASE_      ,  SCREAMING_SNAKE_CASE_      ,  SCREAMING_SNAKE_CASE_							)
																						lowerCAmelCase_       :					Union[str, Any]			 = nn.functional.log_softmax(SCREAMING_SNAKE_CASE_      ,  dim=1							)
																						lowerCAmelCase_       :					List[Any]			 = self.cutoffs[0] + i - 1  # No probability for the head cluster
																						if labels is not None:
																										lowerCAmelCase_       :					Any			 = head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
																										    1      ,  target_i[:, None]							).squeeze(1							)
																						else:
																										lowerCAmelCase_       :					Optional[Any]			 = head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
																										lowerCAmelCase_       :					int			 = logprob_i

																		if labels is not None:
																						if (hasattr(self      ,  '''keep_order'''							) and self.keep_order) or keep_order:
																										out.index_copy_(0      ,  SCREAMING_SNAKE_CASE_      ,  -logprob_i							)
																						else:
																										out[offset : offset + logprob_i.size(0							)].copy_(-logprob_i							)
																						offset += logprob_i.size(0							)

										return out




						def   lowercase_       ( self      ,  __lowercase							)    ->			Optional[int]:
										if self.n_clusters == 0:
														lowerCAmelCase_       :					List[str]			 = self._compute_logit(SCREAMING_SNAKE_CASE_      ,  self.out_layers[0].weight      ,  self.out_layers[0].bias      ,  self.out_projs[0]							)
														return nn.functional.log_softmax(SCREAMING_SNAKE_CASE_      ,  dim=-1							)
										else:
														# construct weights and biases
														lowerCAmelCase_ ,     lowerCAmelCase_       :					List[Any]			 = [], []
														for i in range(len(self.cutoffs							)							):
																		if self.div_val == 1:
																						lowerCAmelCase_ ,     lowerCAmelCase_       :					str			 = self.cutoff_ends[i], self.cutoff_ends[i + 1]
																						lowerCAmelCase_       :					int			 = self.out_layers[0].weight[l_idx:r_idx]
																						lowerCAmelCase_       :					Optional[Any]			 = self.out_layers[0].bias[l_idx:r_idx]
																		else:
																						lowerCAmelCase_       :					List[str]			 = self.out_layers[i].weight
																						lowerCAmelCase_       :					List[Any]			 = self.out_layers[i].bias

																		if i == 0:
																						lowerCAmelCase_       :					Optional[int]			 = torch.cat([weight_i, self.cluster_weight]      ,  dim=0							)
																						lowerCAmelCase_       :					Union[str, Any]			 = torch.cat([bias_i, self.cluster_bias]      ,  dim=0							)

																		weights.append(SCREAMING_SNAKE_CASE_							)
																		biases.append(SCREAMING_SNAKE_CASE_							)

														lowerCAmelCase_ ,     lowerCAmelCase_ ,     lowerCAmelCase_       :					List[Any]			 = weights[0], biases[0], self.out_projs[0]
														lowerCAmelCase_       :					Tuple			 = self._compute_logit(SCREAMING_SNAKE_CASE_      ,  SCREAMING_SNAKE_CASE_      ,  SCREAMING_SNAKE_CASE_      ,  SCREAMING_SNAKE_CASE_							)

														lowerCAmelCase_       :					List[str]			 = hidden.new_empty((head_logit.size(0							), self.n_token)							)
														lowerCAmelCase_       :					Dict			 = nn.functional.log_softmax(SCREAMING_SNAKE_CASE_      ,  dim=1							)

														lowerCAmelCase_       :					Union[str, Any]			 = [0] + self.cutoffs
														for i in range(len(SCREAMING_SNAKE_CASE_							) - 1							):
																		lowerCAmelCase_ ,     lowerCAmelCase_       :					int			 = cutoff_values[i], cutoff_values[i + 1]

																		if i == 0:
																						lowerCAmelCase_       :					Optional[Any]			 = head_logprob[:, : self.cutoffs[0]]
																		else:
																						lowerCAmelCase_ ,     lowerCAmelCase_ ,     lowerCAmelCase_       :					Tuple			 = weights[i], biases[i], self.out_projs[i]

																						lowerCAmelCase_       :					Optional[Any]			 = self._compute_logit(SCREAMING_SNAKE_CASE_      ,  SCREAMING_SNAKE_CASE_      ,  SCREAMING_SNAKE_CASE_      ,  SCREAMING_SNAKE_CASE_							)
																						lowerCAmelCase_       :					Dict			 = nn.functional.log_softmax(SCREAMING_SNAKE_CASE_      ,  dim=1							)

																						lowerCAmelCase_       :					Any			 = head_logprob[:, -i] + tail_logprob_i
																						lowerCAmelCase_       :					List[Any]			 = logprob_i

														return out

import logging

from transformers.configuration_utils import PretrainedConfig


__a  			=					logging.getLogger(__name__)
class  lowercase__(				UpperCAmelCase							):



	"""simple docstring"""




	a			:Optional[int]         =	'masked_bert'
	def __init__(       self  :			Optional[int]       ,							SCREAMING_SNAKE_CASE_  :			List[Any]=3_0_5_2_2       ,							SCREAMING_SNAKE_CASE_  :			List[str]=7_6_8       ,							SCREAMING_SNAKE_CASE_  :			Optional[int]=1_2       ,							SCREAMING_SNAKE_CASE_  :			Any=1_2       ,							SCREAMING_SNAKE_CASE_  :			str=3_0_7_2       ,							SCREAMING_SNAKE_CASE_  :			Union[str, Any]="gelu"       ,							SCREAMING_SNAKE_CASE_  :			Optional[int]=0.1       ,							SCREAMING_SNAKE_CASE_  :			List[str]=0.1       ,							SCREAMING_SNAKE_CASE_  :			Tuple=5_1_2       ,							SCREAMING_SNAKE_CASE_  :			str=2       ,							SCREAMING_SNAKE_CASE_  :			Dict=0.02       ,							SCREAMING_SNAKE_CASE_  :			Any=1e-12       ,							SCREAMING_SNAKE_CASE_  :			Any=0       ,							SCREAMING_SNAKE_CASE_  :			Optional[int]="topK"       ,							SCREAMING_SNAKE_CASE_  :			Dict="constant"       ,							SCREAMING_SNAKE_CASE_  :			Optional[Any]=0.0       ,							**SCREAMING_SNAKE_CASE_  :			Optional[int]       ,							)       ->     Optional[Any]:
					super().__init__(pad_token_id=SCREAMING_SNAKE_CASE_       ,							**SCREAMING_SNAKE_CASE_       )

					lowercase_       =					vocab_size
					lowercase_       =					hidden_size
					lowercase_       =					num_hidden_layers
					lowercase_       =					num_attention_heads
					lowercase_       =					hidden_act
					lowercase_       =					intermediate_size
					lowercase_       =					hidden_dropout_prob
					lowercase_       =					attention_probs_dropout_prob
					lowercase_       =					max_position_embeddings
					lowercase_       =					type_vocab_size
					lowercase_       =					initializer_range
					lowercase_       =					layer_norm_eps
					lowercase_       =					pruning_method
					lowercase_       =					mask_init
					lowercase_       =					mask_scale




