






'''simple docstring'''







import torch
from torch import nn

class _lowercase	(				nn.Module		):




							def __init__(							self:				Optional[Any]			,							UpperCamelCase__:				str			,							UpperCamelCase__:				Union[str, Any]			,							UpperCamelCase__:				Tuple			,							UpperCamelCase__:				Any			,							UpperCamelCase__:				int=1			,							UpperCamelCase__:				List[str]=False     ):
														super().__init__()

														lowerCamelCase__					:	Union[str, Any]         =       n_token
														lowerCamelCase__					:	List[str]         =       d_embed
														lowerCamelCase__					:	Dict         =       d_proj

														lowerCamelCase__					:	Optional[Any]         =       cutoffs + [n_token]
														lowerCamelCase__					:	List[Any]         =       [0] + self.cutoffs
														lowerCamelCase__					:	Tuple         =       div_val

														lowerCamelCase__					:	str         =       self.cutoffs[0]
														lowerCamelCase__					:	int         =       len(self.cutoffs     ) - 1
														lowerCamelCase__					:	List[str]         =       self.shortlist_size + self.n_clusters

														if self.n_clusters > 0:
																					lowerCamelCase__					:	Optional[Any]         =       nn.Parameter(torch.zeros(self.n_clusters			,							self.d_embed     )     )
																					lowerCamelCase__					:	Dict         =       nn.Parameter(torch.zeros(self.n_clusters     )     )

														lowerCamelCase__					:	Dict         =       nn.ModuleList()
														lowerCamelCase__					:	Optional[Any]         =       nn.ParameterList()

														if div_val == 1:
																					for i in range(len(self.cutoffs     )     ):
																												if d_proj != d_embed:
																																			self.out_projs.append(nn.Parameter(torch.FloatTensor(UpperCamelCase__			,							UpperCamelCase__     )     )     )
																												else:
																																			self.out_projs.append(UpperCamelCase__     )

																					self.out_layers.append(nn.Linear(UpperCamelCase__			,							UpperCamelCase__     )     )
														else:
																					for i in range(len(self.cutoffs     )     ):
																												lowerCamelCase__  ,		lowerCamelCase__					:	str         =       self.cutoff_ends[i], self.cutoff_ends[i + 1]
																												lowerCamelCase__					:	int         =       d_embed // (div_val**i)

																												self.out_projs.append(nn.Parameter(torch.FloatTensor(UpperCamelCase__			,							UpperCamelCase__     )     )     )

																												self.out_layers.append(nn.Linear(UpperCamelCase__			,							r_idx - l_idx     )     )

														lowerCamelCase__					:	Tuple         =       keep_order




							def 		lowerCamelCase_				(							self:				str			,							UpperCamelCase__:				str			,							UpperCamelCase__:				Dict			,							UpperCamelCase__:				Union[str, Any]			,							UpperCamelCase__:				Union[str, Any]     ):







														if proj is None:
																					lowerCamelCase__					:	List[Any]         =       nn.functional.linear(UpperCamelCase__			,							UpperCamelCase__			,							bias=UpperCamelCase__     )
														else:
																					# if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
																					lowerCamelCase__					:	int         =       nn.functional.linear(UpperCamelCase__			,							proj.t().contiguous()     )
																					lowerCamelCase__					:	Dict         =       nn.functional.linear(UpperCamelCase__			,							UpperCamelCase__			,							bias=UpperCamelCase__     )
																					# else:
																					#     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
																					#     if bias is not None:
																					#         logit = logit + bias

														return logit




							def 		lowerCamelCase_				(							self:				Optional[Any]			,							UpperCamelCase__:				List[Any]			,							UpperCamelCase__:				List[str]=None			,							UpperCamelCase__:				Tuple=False     ):







														if labels is not None:
																					# Shift so that tokens < n predict n
																					lowerCamelCase__					:	Dict         =       hidden[..., :-1, :].contiguous()
																					lowerCamelCase__					:	List[Any]         =       labels[..., 1:].contiguous()
																					lowerCamelCase__					:	List[Any]         =       hidden.view(-1			,							hidden.size(-1     )     )
																					lowerCamelCase__					:	int         =       labels.view(-1     )
																					if hidden.size(0     ) != labels.size(0     ):
																												raise RuntimeError("""Input and labels should have the same size in the batch dimension."""     )
														else:
																					lowerCamelCase__					:	Optional[Any]         =       hidden.view(-1			,							hidden.size(-1     )     )

														if self.n_clusters == 0:
																					lowerCamelCase__					:	str         =       self._compute_logit(UpperCamelCase__			,							self.out_layers[0].weight			,							self.out_layers[0].bias			,							self.out_projs[0]     )
																					if labels is not None:
																												lowerCamelCase__					:	Optional[Any]         =       labels != -100
																												lowerCamelCase__					:	Tuple         =       torch.zeros_like(UpperCamelCase__			,							dtype=hidden.dtype			,							device=hidden.device     )
																												lowerCamelCase__					:	Union[str, Any]         =       (
																												    -nn.functional.log_softmax(UpperCamelCase__			,							dim=-1     )[mask].gather(1			,							labels[mask].unsqueeze(1     )     ).squeeze(1     )
																												)
																					else:
																												lowerCamelCase__					:	Union[str, Any]         =       nn.functional.log_softmax(UpperCamelCase__			,							dim=-1     )
														else:
																					# construct weights and biases
																					lowerCamelCase__  ,		lowerCamelCase__					:	Tuple         =       [], []
																					for i in range(len(self.cutoffs     )     ):
																												if self.div_val == 1:
																																			lowerCamelCase__  ,		lowerCamelCase__					:	str         =       self.cutoff_ends[i], self.cutoff_ends[i + 1]
																																			lowerCamelCase__					:	List[Any]         =       self.out_layers[0].weight[l_idx:r_idx]
																																			lowerCamelCase__					:	str         =       self.out_layers[0].bias[l_idx:r_idx]
																												else:
																																			lowerCamelCase__					:	Union[str, Any]         =       self.out_layers[i].weight
																																			lowerCamelCase__					:	Tuple         =       self.out_layers[i].bias

																												if i == 0:
																																			lowerCamelCase__					:	int         =       torch.cat([weight_i, self.cluster_weight]			,							dim=0     )
																																			lowerCamelCase__					:	Union[str, Any]         =       torch.cat([bias_i, self.cluster_bias]			,							dim=0     )

																												weights.append(UpperCamelCase__     )
																												biases.append(UpperCamelCase__     )

																					lowerCamelCase__  ,		lowerCamelCase__  ,		lowerCamelCase__					:	List[Any]         =       weights[0], biases[0], self.out_projs[0]

																					lowerCamelCase__					:	int         =       self._compute_logit(UpperCamelCase__			,							UpperCamelCase__			,							UpperCamelCase__			,							UpperCamelCase__     )
																					lowerCamelCase__					:	Any         =       nn.functional.log_softmax(UpperCamelCase__			,							dim=1     )

																					if labels is None:
																												lowerCamelCase__					:	Optional[int]         =       hidden.new_empty((head_logit.size(0     ), self.n_token)     )
																					else:
																												lowerCamelCase__					:	str         =       torch.zeros_like(UpperCamelCase__			,							dtype=hidden.dtype			,							device=hidden.device     )

																					lowerCamelCase__					:	Union[str, Any]         =       0
																					lowerCamelCase__					:	Union[str, Any]         =       [0] + self.cutoffs
																					for i in range(len(UpperCamelCase__     ) - 1     ):
																												lowerCamelCase__  ,		lowerCamelCase__					:	Tuple         =       cutoff_values[i], cutoff_values[i + 1]

																												if labels is not None:
																																			lowerCamelCase__					:	int         =       (labels >= l_idx) & (labels < r_idx)
																																			lowerCamelCase__					:	Union[str, Any]         =       mask_i.nonzero().squeeze()

																																			if indices_i.numel() == 0:
																																										continue

																																			lowerCamelCase__					:	Optional[int]         =       labels.index_select(0			,							UpperCamelCase__     ) - l_idx
																																			lowerCamelCase__					:	Optional[int]         =       head_logprob.index_select(0			,							UpperCamelCase__     )
																																			lowerCamelCase__					:	str         =       hidden.index_select(0			,							UpperCamelCase__     )
																												else:
																																			lowerCamelCase__					:	str         =       hidden

																												if i == 0:
																																			if labels is not None:
																																										lowerCamelCase__					:	List[Any]         =       head_logprob_i.gather(1			,							target_i[:, None]     ).squeeze(1     )
																																			else:
																																										lowerCamelCase__					:	Tuple         =       head_logprob[:, : self.cutoffs[0]]
																												else:
																																			lowerCamelCase__  ,		lowerCamelCase__  ,		lowerCamelCase__					:	int         =       weights[i], biases[i], self.out_projs[i]

																																			lowerCamelCase__					:	Any         =       self._compute_logit(UpperCamelCase__			,							UpperCamelCase__			,							UpperCamelCase__			,							UpperCamelCase__     )
																																			lowerCamelCase__					:	Union[str, Any]         =       nn.functional.log_softmax(UpperCamelCase__			,							dim=1     )
																																			lowerCamelCase__					:	Optional[Any]         =       self.cutoffs[0] + i - 1  # No probability for the head cluster
																																			if labels is not None:
																																										lowerCamelCase__					:	Union[str, Any]         =       head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
																																										    1			,							target_i[:, None]     ).squeeze(1     )
																																			else:
																																										lowerCamelCase__					:	Optional[Any]         =       head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
																																										lowerCamelCase__					:	Dict         =       logprob_i

																												if labels is not None:
																																			if (hasattr(self			,							"""keep_order"""     ) and self.keep_order) or keep_order:
																																										out.index_copy_(0			,							UpperCamelCase__			,							-logprob_i     )
																																			else:
																																										out[offset : offset + logprob_i.size(0     )].copy_(-logprob_i     )
																																			offset += logprob_i.size(0     )

														return out




							def 		lowerCamelCase_				(							self:				List[str]			,							UpperCamelCase__:				Tuple     ):







														if self.n_clusters == 0:
																					lowerCamelCase__					:	Tuple         =       self._compute_logit(UpperCamelCase__			,							self.out_layers[0].weight			,							self.out_layers[0].bias			,							self.out_projs[0]     )
																					return nn.functional.log_softmax(UpperCamelCase__			,							dim=-1     )
														else:
																					# construct weights and biases
																					lowerCamelCase__  ,		lowerCamelCase__					:	Optional[Any]         =       [], []
																					for i in range(len(self.cutoffs     )     ):
																												if self.div_val == 1:
																																			lowerCamelCase__  ,		lowerCamelCase__					:	Dict         =       self.cutoff_ends[i], self.cutoff_ends[i + 1]
																																			lowerCamelCase__					:	int         =       self.out_layers[0].weight[l_idx:r_idx]
																																			lowerCamelCase__					:	List[Any]         =       self.out_layers[0].bias[l_idx:r_idx]
																												else:
																																			lowerCamelCase__					:	List[str]         =       self.out_layers[i].weight
																																			lowerCamelCase__					:	Optional[Any]         =       self.out_layers[i].bias

																												if i == 0:
																																			lowerCamelCase__					:	Union[str, Any]         =       torch.cat([weight_i, self.cluster_weight]			,							dim=0     )
																																			lowerCamelCase__					:	str         =       torch.cat([bias_i, self.cluster_bias]			,							dim=0     )

																												weights.append(UpperCamelCase__     )
																												biases.append(UpperCamelCase__     )

																					lowerCamelCase__  ,		lowerCamelCase__  ,		lowerCamelCase__					:	Tuple         =       weights[0], biases[0], self.out_projs[0]
																					lowerCamelCase__					:	Tuple         =       self._compute_logit(UpperCamelCase__			,							UpperCamelCase__			,							UpperCamelCase__			,							UpperCamelCase__     )

																					lowerCamelCase__					:	int         =       hidden.new_empty((head_logit.size(0     ), self.n_token)     )
																					lowerCamelCase__					:	Optional[Any]         =       nn.functional.log_softmax(UpperCamelCase__			,							dim=1     )

																					lowerCamelCase__					:	List[str]         =       [0] + self.cutoffs
																					for i in range(len(UpperCamelCase__     ) - 1     ):
																												lowerCamelCase__  ,		lowerCamelCase__					:	Tuple         =       cutoff_values[i], cutoff_values[i + 1]

																												if i == 0:
																																			lowerCamelCase__					:	Union[str, Any]         =       head_logprob[:, : self.cutoffs[0]]
																												else:
																																			lowerCamelCase__  ,		lowerCamelCase__  ,		lowerCamelCase__					:	List[Any]         =       weights[i], biases[i], self.out_projs[i]

																																			lowerCamelCase__					:	List[Any]         =       self._compute_logit(UpperCamelCase__			,							UpperCamelCase__			,							UpperCamelCase__			,							UpperCamelCase__     )
																																			lowerCamelCase__					:	Optional[int]         =       nn.functional.log_softmax(UpperCamelCase__			,							dim=1     )

																																			lowerCamelCase__					:	List[str]         =       head_logprob[:, -i] + tail_logprob_i
																																			lowerCamelCase__					:	int         =       logprob_i

																					return out





'''simple docstring'''

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase							:   Tuple  						=	logging.get_logger(__name__)

lowerCamelCase							:   Dict  						=	{
    'microsoft/cvt-13': 'https://huggingface.co/microsoft/cvt-13/resolve/main/config.json',
    # See all Cvt models at https://huggingface.co/models?filter=cvt
}
class   __lowerCAmelCase						(lowercase_     ):





      '''simple docstring'''






      lowerCAmelCase__					:     Tuple  					=							"""cvt"""



      def __init__(self    :    int   ,						UpperCamelCase    :    List[Any]=3   ,						UpperCamelCase    :    int=[7, 3, 3]   ,						UpperCamelCase    :    str=[4, 2, 2]   ,						UpperCamelCase    :    Dict=[2, 1, 1]   ,						UpperCamelCase    :    Dict=[64, 192, 384]   ,						UpperCamelCase    :    Dict=[1, 3, 6]   ,						UpperCamelCase    :    Dict=[1, 2, 10]   ,						UpperCamelCase    :    Any=[4.0, 4.0, 4.0]   ,						UpperCamelCase    :    Optional[int]=[0.0, 0.0, 0.0]   ,						UpperCamelCase    :    Optional[int]=[0.0, 0.0, 0.0]   ,						UpperCamelCase    :    int=[0.0, 0.0, 0.1]   ,						UpperCamelCase    :    Any=[True, True, True]   ,						UpperCamelCase    :    int=[False, False, True]   ,						UpperCamelCase    :    Union[str, Any]=["dw_bn", "dw_bn", "dw_bn"]   ,						UpperCamelCase    :    Optional[int]=[3, 3, 3]   ,						UpperCamelCase    :    Tuple=[1, 1, 1]   ,						UpperCamelCase    :    Any=[2, 2, 2]   ,						UpperCamelCase    :    Dict=[1, 1, 1]   ,						UpperCamelCase    :    List[str]=[1, 1, 1]   ,						UpperCamelCase    :    str=0.02   ,						UpperCamelCase    :    int=1E-12   ,						**UpperCamelCase    :    Union[str, Any]   ,						):

             '''simple docstring'''







             super().__init__(**UpperCamelCase )
             lowercase__      			= num_channels
             lowercase__      			= patch_sizes
             lowercase__      			= patch_stride
             lowercase__      			= patch_padding
             lowercase__      			= embed_dim
             lowercase__      			= num_heads
             lowercase__      			= depth
             lowercase__      			= mlp_ratio
             lowercase__      			= attention_drop_rate
             lowercase__      			= drop_rate
             lowercase__      			= drop_path_rate
             lowercase__      			= qkv_bias
             lowercase__      			= cls_token
             lowercase__      			= qkv_projection_method
             lowercase__      			= kernel_qkv
             lowercase__      			= padding_kv
             lowercase__      			= stride_kv
             lowercase__      			= padding_q
             lowercase__      			= stride_q
             lowercase__      			= initializer_range
             lowercase__      			= layer_norm_eps




