





"""simple docstring"""





import math

import torch
from torch import nn
from torch.nn import init

from .binarizer import MagnitudeBinarizer, ThresholdBinarizer, TopKBinarizer





class 							_lowerCamelCase   (							nn.Linear			):


						def __init__(self ,				__a ,				__a ,				__a = True ,				__a = "constant" ,				__a = 0.0 ,				__a = "topK" ,				)			-> str:
								super(__a ,				self		).__init__(in_features=__a ,				out_features=__a ,				bias=__a		)
								assert pruning_method in ["topK", "threshold", "sigmoied_threshold", "magnitude", "l0"]
								UpperCamelCase =						pruning_method

								if self.pruning_method in ["topK", "threshold", "sigmoied_threshold", "l0"]:
										UpperCamelCase =						mask_scale
										UpperCamelCase =						mask_init
										UpperCamelCase =						nn.Parameter(torch.empty(self.weight.size()		)		)
										self.init_mask()


						def      snake_case_ (self		)			-> Tuple:
								if self.mask_init == "constant":
										init.constant_(self.mask_scores ,				val=self.mask_scale		)
								elif self.mask_init == "uniform":
										init.uniform_(self.mask_scores ,				a=-self.mask_scale ,				b=self.mask_scale		)
								elif self.mask_init == "kaiming":
										init.kaiming_uniform_(self.mask_scores ,				a=math.sqrt(5		)		)




						def      snake_case_ (self ,				__a ,				__a		)			-> Optional[int]:
								# Get the mask
								if self.pruning_method == "topK":
										UpperCamelCase =						TopKBinarizer.apply(self.mask_scores ,				__a		)
								elif self.pruning_method in ["threshold", "sigmoied_threshold"]:
										UpperCamelCase =						"sigmoied" in self.pruning_method
										UpperCamelCase =						ThresholdBinarizer.apply(self.mask_scores ,				__a ,				__a		)
								elif self.pruning_method == "magnitude":
										UpperCamelCase =						MagnitudeBinarizer.apply(self.weight ,				__a		)
								elif self.pruning_method == "l0":
										UpperCamelCase ,					UpperCamelCase ,					UpperCamelCase =						-0.1, 1.1, 2 / 3
										if self.training:
												UpperCamelCase =						torch.zeros_like(self.mask_scores		).uniform_().clamp(0.0001 ,				0.9999		)
												UpperCamelCase =						torch.sigmoid((u.log() - (1 - u).log() + self.mask_scores) / b		)
										else:
												UpperCamelCase =						torch.sigmoid(self.mask_scores		)
										UpperCamelCase =						s * (r - l) + l
										UpperCamelCase =						s_bar.clamp(min=0.0 ,				max=1.0		)
								# Mask weights with computed mask
								UpperCamelCase =						mask * self.weight
								# Compute output (linear layer) with masked weights
								return nn.functional.linear(__a ,				__a ,				self.bias		)













"""simple docstring"""





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCAmelCase__        =      {
    '''configuration_openai''': ['''OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''OpenAIGPTConfig'''],
    '''tokenization_openai''': ['''OpenAIGPTTokenizer'''],
}

try:
						if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						lowerCAmelCase__        =      ['''OpenAIGPTTokenizerFast''']

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						lowerCAmelCase__        =      [
						    '''OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST''',
						    '''OpenAIGPTDoubleHeadsModel''',
						    '''OpenAIGPTForSequenceClassification''',
						    '''OpenAIGPTLMHeadModel''',
						    '''OpenAIGPTModel''',
						    '''OpenAIGPTPreTrainedModel''',
						    '''load_tf_weights_in_openai_gpt''',
						]

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						lowerCAmelCase__        =      [
						    '''TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST''',
						    '''TFOpenAIGPTDoubleHeadsModel''',
						    '''TFOpenAIGPTForSequenceClassification''',
						    '''TFOpenAIGPTLMHeadModel''',
						    '''TFOpenAIGPTMainLayer''',
						    '''TFOpenAIGPTModel''',
						    '''TFOpenAIGPTPreTrainedModel''',
						]


if TYPE_CHECKING:
						from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig
						from .tokenization_openai import OpenAIGPTTokenizer

						try:
												if not is_tokenizers_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_openai_fast import OpenAIGPTTokenizerFast

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_openai import (
												    OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    OpenAIGPTDoubleHeadsModel,
												    OpenAIGPTForSequenceClassification,
												    OpenAIGPTLMHeadModel,
												    OpenAIGPTModel,
												    OpenAIGPTPreTrainedModel,
												    load_tf_weights_in_openai_gpt,
												)

						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_tf_openai import (
												    TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    TFOpenAIGPTDoubleHeadsModel,
												    TFOpenAIGPTForSequenceClassification,
												    TFOpenAIGPTLMHeadModel,
												    TFOpenAIGPTMainLayer,
												    TFOpenAIGPTModel,
												    TFOpenAIGPTPreTrainedModel,
												)

else:
						import sys

						lowerCAmelCase__        =      _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)






