


'''simple docstring'''


import torch

from transformers import PreTrainedModel

from .custom_configuration import CustomConfig, NoSuperInitConfig






class 						lowerCAmelCase__      (      UpperCAmelCase__						):
			lowerCAmelCase			:						Union[str, Any]							 =  CustomConfig




			def __init__(		self	:  Union[str, Any]						,	lowerCamelCase__	:  Tuple			) ->List[Any]:



					'''simple docstring'''







					super().__init__(lowerCamelCase__			)
					_UpperCAmelCase		:							Tuple             =   torch.nn.Linear(config.hidden_size						,	config.hidden_size			)




			def 				lowerCAmelCase__     (		self	:  Union[str, Any]						,	lowerCamelCase__	:  str			) ->Dict:



					'''simple docstring'''







					return self.linear(lowerCamelCase__			)






			def 				lowerCAmelCase__     (		self	:  Union[str, Any]						,	lowerCamelCase__	:  int			) ->Dict:



					'''simple docstring'''







					pass






class 						lowerCAmelCase__      (      UpperCAmelCase__						):
			lowerCAmelCase			:						Optional[int]							 =  NoSuperInitConfig




			def __init__(		self	:  Optional[Any]						,	lowerCamelCase__	:  List[str]			) ->Union[str, Any]:



					'''simple docstring'''







					super().__init__(lowerCamelCase__			)
					_UpperCAmelCase		:							Union[str, Any]             =   torch.nn.Linear(config.attribute						,	config.attribute			)




			def 				lowerCAmelCase__     (		self	:  str						,	lowerCamelCase__	:  Union[str, Any]			) ->Optional[Any]:



					'''simple docstring'''







					return self.linear(lowerCamelCase__			)






			def 				lowerCAmelCase__     (		self	:  Tuple						,	lowerCamelCase__	:  Optional[Any]			) ->Dict:



					'''simple docstring'''







					pass







'''simple docstring'''


import torch
from torch import autograd






class 						lowerCAmelCase__      (      autograd.Function						):




			@staticmethod
			def 				lowerCAmelCase__     (		lowerCamelCase__	:  Optional[int]						,	lowerCamelCase__	:  torch.tensor						,	lowerCamelCase__	:  float						,	lowerCamelCase__	:  bool			) ->Optional[Any]:



					'''simple docstring'''







					_UpperCAmelCase		:							Optional[int]             =   inputs.numel()
					_UpperCAmelCase		:							Any             =   int(0.0_0_5 * nb_elems			) + 1
					if sigmoid:
							_UpperCAmelCase		:							List[str]             =   (torch.sigmoid(lowerCamelCase__			) > threshold).type(inputs.type()			)
					else:
							_UpperCAmelCase		:							List[Any]             =   (inputs > threshold).type(inputs.type()			)
					if mask.sum() < nb_min:
							# We limit the pruning so that at least 0.5% (half a percent) of the weights are remaining
							_UpperCAmelCase		:							str             =   inputs.flatten().kthvalue(max(nb_elems - nb_min						,	1			)			).values
							_UpperCAmelCase		:							str             =   (inputs > k_threshold).type(inputs.type()			)
					return mask






			@staticmethod
			def 				lowerCAmelCase__     (		lowerCamelCase__	:  List[Any]						,	lowerCamelCase__	:  Optional[int]			) ->Dict:



					'''simple docstring'''







					return gradOutput, None, None






class 						lowerCAmelCase__      (      autograd.Function						):




			@staticmethod
			def 				lowerCAmelCase__     (		lowerCamelCase__	:  Optional[int]						,	lowerCamelCase__	:  torch.tensor						,	lowerCamelCase__	:  float			) ->List[str]:



					'''simple docstring'''







					_UpperCAmelCase		:							Optional[int]             =   inputs.clone()
					_UpperCAmelCase       ,     _UpperCAmelCase		:							Tuple             =   inputs.flatten().sort(descending=lowerCamelCase__			)
					_UpperCAmelCase		:							Optional[int]             =   int(threshold * inputs.numel()			)

					# flat_out and mask access the same memory.
					_UpperCAmelCase		:							str             =   mask.flatten()
					_UpperCAmelCase		:							Optional[int]             =   0
					_UpperCAmelCase		:							Optional[Any]             =   1
					return mask






			@staticmethod
			def 				lowerCAmelCase__     (		lowerCamelCase__	:  Optional[int]						,	lowerCamelCase__	:  Any			) ->Union[str, Any]:



					'''simple docstring'''







					return gradOutput, None






class 						lowerCAmelCase__      (      UpperCAmelCase__						):




			@staticmethod
			def 				lowerCAmelCase__     (		lowerCamelCase__	:  torch.tensor						,	lowerCamelCase__	:  float			) ->Tuple:



					'''simple docstring'''







					_UpperCAmelCase		:							List[Any]             =   inputs.clone()
					_UpperCAmelCase       ,     _UpperCAmelCase		:							Optional[Any]             =   inputs.abs().flatten().sort(descending=lowerCamelCase__			)
					_UpperCAmelCase		:							int             =   int(threshold * inputs.numel()			)

					# flat_out and mask access the same memory.
					_UpperCAmelCase		:							int             =   mask.flatten()
					_UpperCAmelCase		:							List[Any]             =   0
					_UpperCAmelCase		:							int             =   1
					return mask



