




'''simple docstring'''

import unittest

from transformers import MPNetConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MPNetForMaskedLM,
							    MPNetForMultipleChoice,
							    MPNetForQuestionAnswering,
							    MPNetForSequenceClassification,
							    MPNetForTokenClassification,
							    MPNetModel,
							)






class 						a				:


				def __init__(		self    :					List[Any]       ,				lowercase_    :					Optional[Any]       ,				lowercase_    :					Any=13       ,				lowercase_    :					Optional[Any]=7       ,				lowercase_    :					Optional[Any]=True       ,				lowercase_    :					Dict=True       ,				lowercase_    :					Tuple=False       ,				lowercase_    :					Optional[Any]=True       ,				lowercase_    :					Any=99       ,				lowercase_    :					Union[str, Any]=64       ,				lowercase_    :					str=5       ,				lowercase_    :					int=4       ,				lowercase_    :					List[Any]=64       ,				lowercase_    :					Dict="gelu"       ,				lowercase_    :					Optional[int]=0.1       ,				lowercase_    :					Optional[Any]=0.1       ,				lowercase_    :					Tuple=512       ,				lowercase_    :					List[Any]=16       ,				lowercase_    :					str=2       ,				lowercase_    :					List[str]=0.02       ,				lowercase_    :					Optional[Any]=3       ,				lowercase_    :					Optional[Any]=4       ,				lowercase_    :					List[Any]=None       ,				):
									snake_case_									=				parent
									snake_case_									=				batch_size
									snake_case_									=				seq_length
									snake_case_									=				is_training
									snake_case_									=				use_input_mask
									snake_case_									=				use_token_type_ids
									snake_case_									=				use_labels
									snake_case_									=				vocab_size
									snake_case_									=				hidden_size
									snake_case_									=				num_hidden_layers
									snake_case_									=				num_attention_heads
									snake_case_									=				intermediate_size
									snake_case_									=				hidden_act
									snake_case_									=				hidden_dropout_prob
									snake_case_									=				attention_probs_dropout_prob
									snake_case_									=				max_position_embeddings
									snake_case_									=				type_vocab_size
									snake_case_									=				type_sequence_label_size
									snake_case_									=				initializer_range
									snake_case_									=				num_labels
									snake_case_									=				num_choices
									snake_case_									=				scope


				def   A_ (		self    :					List[str]					):
									return MPNetConfig.from_pretrained('''microsoft/mpnet-base'''					)


				def   A_ (		self    :					str					):
									snake_case_									=				ids_tensor([self.batch_size, self.seq_length]       ,				self.vocab_size					)

									snake_case_									=				None
									if self.use_input_mask:
														snake_case_									=				random_attention_mask([self.batch_size, self.seq_length]					)

									snake_case_									=				None
									snake_case_									=				None
									snake_case_									=				None
									if self.use_labels:
														snake_case_									=				ids_tensor([self.batch_size]       ,				self.type_sequence_label_size					)
														snake_case_									=				ids_tensor([self.batch_size, self.seq_length]       ,				self.num_labels					)
														snake_case_									=				ids_tensor([self.batch_size]       ,				self.num_choices					)

									snake_case_									=				self.get_config()
									return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels


				def   A_ (		self    :					Tuple					):
									return MPNetConfig(
									    vocab_size=self.vocab_size       ,				hidden_size=self.hidden_size       ,				num_hidden_layers=self.num_hidden_layers       ,				num_attention_heads=self.num_attention_heads       ,				intermediate_size=self.intermediate_size       ,				hidden_act=self.hidden_act       ,				hidden_dropout_prob=self.hidden_dropout_prob       ,				attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,				max_position_embeddings=self.max_position_embeddings       ,				initializer_range=self.initializer_range       ,				)


				def   A_ (		self    :					Any       ,				lowercase_    :					Dict       ,				lowercase_    :					str       ,				lowercase_    :					Optional[Any]       ,				lowercase_    :					Union[str, Any]       ,				lowercase_    :					Dict       ,				lowercase_    :					Optional[int]					):
									snake_case_									=				MPNetModel(config=lowercase_					)
									model.to(lowercase_					)
									model.eval()
									snake_case_									=				model(lowercase_       ,				lowercase_					)
									snake_case_									=				model(lowercase_					)
									self.parent.assertEqual(result.last_hidden_state.shape       ,				(self.batch_size, self.seq_length, self.hidden_size)					)
									self.parent.assertEqual(result.pooler_output.shape       ,				(self.batch_size, self.hidden_size)					)


				def   A_ (		self    :					str       ,				lowercase_    :					Optional[Any]       ,				lowercase_    :					Any       ,				lowercase_    :					str       ,				lowercase_    :					Tuple       ,				lowercase_    :					Tuple       ,				lowercase_    :					Optional[int]					):
									snake_case_									=				MPNetForQuestionAnswering(config=lowercase_					)
									model.to(lowercase_					)
									model.eval()
									snake_case_									=				model(
									    lowercase_       ,				attention_mask=lowercase_       ,				start_positions=lowercase_       ,				end_positions=lowercase_       ,				)
									self.parent.assertEqual(result.start_logits.shape       ,				(self.batch_size, self.seq_length)					)
									self.parent.assertEqual(result.end_logits.shape       ,				(self.batch_size, self.seq_length)					)


				def   A_ (		self    :					Tuple       ,				lowercase_    :					Tuple       ,				lowercase_    :					List[str]       ,				lowercase_    :					str       ,				lowercase_    :					Optional[int]       ,				lowercase_    :					Optional[Any]       ,				lowercase_    :					Any					):
									snake_case_									=				self.num_labels
									snake_case_									=				MPNetForSequenceClassification(lowercase_					)
									model.to(lowercase_					)
									model.eval()
									snake_case_									=				model(lowercase_       ,				attention_mask=lowercase_       ,				labels=lowercase_					)
									self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.num_labels)					)


				def   A_ (		self    :					Any       ,				lowercase_    :					Any       ,				lowercase_    :					str       ,				lowercase_    :					Tuple       ,				lowercase_    :					List[str]       ,				lowercase_    :					Union[str, Any]       ,				lowercase_    :					Dict					):
									snake_case_									=				self.num_choices
									snake_case_									=				MPNetForMultipleChoice(config=lowercase_					)
									model.to(lowercase_					)
									model.eval()
									snake_case_									=				input_ids.unsqueeze(1					).expand(-1       ,				self.num_choices       ,				-1					).contiguous()
									snake_case_									=				input_mask.unsqueeze(1					).expand(-1       ,				self.num_choices       ,				-1					).contiguous()
									snake_case_									=				model(
									    lowercase_       ,				attention_mask=lowercase_       ,				labels=lowercase_       ,				)
									self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.num_choices)					)


				def   A_ (		self    :					Union[str, Any]       ,				lowercase_    :					List[Any]       ,				lowercase_    :					Optional[Any]       ,				lowercase_    :					Optional[int]       ,				lowercase_    :					str       ,				lowercase_    :					int       ,				lowercase_    :					List[str]					):
									snake_case_									=				self.num_labels
									snake_case_									=				MPNetForTokenClassification(config=lowercase_					)
									model.to(lowercase_					)
									model.eval()
									snake_case_									=				model(lowercase_       ,				attention_mask=lowercase_       ,				labels=lowercase_					)
									self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.seq_length, self.num_labels)					)


				def   A_ (		self    :					Union[str, Any]					):
									snake_case_									=				self.prepare_config_and_inputs()
									((snake_case_)			,(snake_case_)			,(snake_case_)			,(snake_case_)			,(snake_case_)			,(snake_case_))									=				config_and_inputs
									snake_case_									=				{'''input_ids''': input_ids, '''attention_mask''': input_mask}
									return config, inputs_dict







@require_torch
class 						a				(   _lowerCamelCase   ,     _lowerCamelCase   ,     unittest.TestCase     ):
				snake_case_      =		(
				    (
				        MPNetForMaskedLM,
				        MPNetForMultipleChoice,
				        MPNetForQuestionAnswering,
				        MPNetForSequenceClassification,
				        MPNetForTokenClassification,
				        MPNetModel,
				    )
				    if is_torch_available()
				    else ()
				)
				snake_case_      =		(
				    {
				        "feature-extraction": MPNetModel,
				        "fill-mask": MPNetForMaskedLM,
				        "question-answering": MPNetForQuestionAnswering,
				        "text-classification": MPNetForSequenceClassification,
				        "token-classification": MPNetForTokenClassification,
				        "zero-shot": MPNetForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				snake_case_      =		False
				snake_case_      =		True


				def   A_ (		self    :					Tuple					):
									snake_case_									=				MPNetModelTester(self					)
									snake_case_									=				ConfigTester(self       ,				config_class=lowercase_       ,				hidden_size=37					)


				def   A_ (		self    :					Union[str, Any]					):
									self.config_tester.run_common_tests()


				def   A_ (		self    :					Tuple					):
									snake_case_									=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mpnet_model(*lowercase_					)


				def   A_ (		self    :					List[Any]					):
									snake_case_									=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mpnet_for_sequence_classification(*lowercase_					)


				def   A_ (		self    :					List[Any]					):
									snake_case_									=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mpnet_for_multiple_choice(*lowercase_					)


				def   A_ (		self    :					Union[str, Any]					):
									snake_case_									=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mpnet_for_token_classification(*lowercase_					)


				def   A_ (		self    :					Tuple					):
									snake_case_									=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mpnet_for_question_answering(*lowercase_					)







@require_torch
class 						a				(   unittest.TestCase     ):


				@slow
				def   A_ (		self    :					List[Any]					):
									snake_case_									=				MPNetModel.from_pretrained('''microsoft/mpnet-base'''					)
									snake_case_									=				torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]					)
									snake_case_									=				model(lowercase_					)[0]
									snake_case_									=				torch.Size((1, 11, 768)					)
									self.assertEqual(output.shape       ,				lowercase_					)
									snake_case_									=				torch.tensor(
									    [[[-0.0550, 0.1943, -0.0740], [-0.0562, 0.2211, -0.0579], [-0.0437, 0.3337, -0.0641]]]					)
									# compare the actual values for a slice.
									self.assertTrue(torch.allclose(output[:, :3, :3]       ,				lowercase_       ,				atol=1e-4					)					)







'''simple docstring'''

import re
from typing import Callable, List, Optional, Union

import tensorflow as tf


try:
							from tensorflow.keras.optimizers.legacy import Adam
except ImportError:
							from tensorflow.keras.optimizers import Adam






class 						a				(   tf.keras.optimizers.schedules.LearningRateSchedule     ):


				def __init__(		self    :					Optional[Any]       ,				lowercase_    :					float       ,				lowercase_    :					Callable       ,				lowercase_    :					int       ,				lowercase_    :					float = 1.0       ,				lowercase_    :					str = None       ,				):
									super().__init__()
									snake_case_									=				initial_learning_rate
									snake_case_									=				warmup_steps
									snake_case_									=				power
									snake_case_									=				decay_schedule_fn
									snake_case_									=				name


				def __call__(		self    :					Tuple       ,				lowercase_    :					str					):
									with tf.name_scope(self.name or '''WarmUp'''					) as name:
														# Implements polynomial warmup. i.e., if global_step < warmup_steps, the
														# learning rate will be `global_step/num_warmup_steps * init_lr`.
														snake_case_									=				tf.cast(lowercase_       ,				tf.floataa					)
														snake_case_									=				tf.cast(self.warmup_steps       ,				tf.floataa					)
														snake_case_									=				global_step_float / warmup_steps_float
														snake_case_									=				self.initial_learning_rate * tf.math.pow(lowercase_       ,				self.power					)
														return tf.cond(
														    global_step_float < warmup_steps_float       ,				lambda: warmup_learning_rate       ,				lambda: self.decay_schedule_fn(step - self.warmup_steps					)       ,				name=lowercase_       ,				)


				def   A_ (		self    :					Any					):
									return {
									    "initial_learning_rate": self.initial_learning_rate,
									    "decay_schedule_fn": self.decay_schedule_fn,
									    "warmup_steps": self.warmup_steps,
									    "power": self.power,
									    "name": self.name,
									}







def 		__magic_name__    (	__UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase = 0.0,      __UpperCAmelCase = 0.9,      __UpperCAmelCase = 0.9_9_9,      __UpperCAmelCase = 1e-8,      __UpperCAmelCase = None,      __UpperCAmelCase = None,      __UpperCAmelCase = 0.0,      __UpperCAmelCase = 1.0,      __UpperCAmelCase = None,      )		->					List[str]:

					'''simple docstring'''

					snake_case_									=				tf.keras.optimizers.schedules.PolynomialDecay(
					    initial_learning_rate=__UpperCAmelCase,      decay_steps=num_train_steps - num_warmup_steps,      end_learning_rate=init_lr * min_lr_ratio,      power=__UpperCAmelCase,      )
					if num_warmup_steps:
										snake_case_									=				WarmUp(
										    initial_learning_rate=__UpperCAmelCase,      decay_schedule_fn=__UpperCAmelCase,      warmup_steps=__UpperCAmelCase,      )
					if weight_decay_rate > 0.0:
										snake_case_									=				AdamWeightDecay(
										    learning_rate=__UpperCAmelCase,      weight_decay_rate=__UpperCAmelCase,      beta_a=__UpperCAmelCase,      beta_a=__UpperCAmelCase,      epsilon=__UpperCAmelCase,      clipnorm=__UpperCAmelCase,      global_clipnorm=__UpperCAmelCase,      exclude_from_weight_decay=['''LayerNorm''', '''layer_norm''', '''bias'''],      include_in_weight_decay=__UpperCAmelCase,      )
					else:
										snake_case_									=				tf.keras.optimizers.Adam(
										    learning_rate=__UpperCAmelCase,      beta_a=__UpperCAmelCase,      beta_a=__UpperCAmelCase,      epsilon=__UpperCAmelCase,      clipnorm=__UpperCAmelCase,      global_clipnorm=__UpperCAmelCase,      )
					# We return the optimizer and the LR scheduler in order to better track the
					# evolution of the LR independently of the optimizer.
					return optimizer, lr_schedule






class 						a				(   _lowerCamelCase     ):


				def __init__(		self    :					Dict       ,				lowercase_    :					Union[float, tf.keras.optimizers.schedules.LearningRateSchedule] = 0.001       ,				lowercase_    :					float = 0.9       ,				lowercase_    :					float = 0.999       ,				lowercase_    :					float = 1e-7       ,				lowercase_    :					bool = False       ,				lowercase_    :					float = 0.0       ,				lowercase_    :					Optional[List[str]] = None       ,				lowercase_    :					Optional[List[str]] = None       ,				lowercase_    :					str = "AdamWeightDecay"       ,				**lowercase_    :					Optional[int]       ,				):
									super().__init__(lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_       ,				**lowercase_					)
									snake_case_									=				weight_decay_rate
									snake_case_									=				include_in_weight_decay
									snake_case_									=				exclude_from_weight_decay


				@classmethod
				def   A_ (		cls    :					Dict       ,				lowercase_    :					Union[str, Any]					):
									snake_case_									=				{'''WarmUp''': WarmUp}
									return super(lowercase_       ,				cls					).from_config(lowercase_       ,				custom_objects=lowercase_					)


				def   A_ (		self    :					str       ,				lowercase_    :					Optional[int]       ,				lowercase_    :					List[str]       ,				lowercase_    :					Optional[int]					):
									super(lowercase_       ,				self					)._prepare_local(lowercase_       ,				lowercase_       ,				lowercase_					)
									snake_case_									=				tf.constant(
									    self.weight_decay_rate       ,				name='''adam_weight_decay_rate'''					)


				def   A_ (		self    :					str       ,				lowercase_    :					Union[str, Any]       ,				lowercase_    :					Dict       ,				lowercase_    :					Any					):
									snake_case_									=				self._do_use_weight_decay(var.name					)
									if do_decay:
														return var.assign_sub(
														    learning_rate * var * apply_state[(var.device, var.dtype.base_dtype)]['''weight_decay_rate''']       ,				use_locking=self._use_locking       ,				)
									return tf.no_op()


				def   A_ (		self    :					Union[str, Any]       ,				lowercase_    :					int       ,				lowercase_    :					str=None       ,				**lowercase_    :					List[str]					):
									snake_case_			,snake_case_									=				list(zip(*lowercase_					)					)
									return super(lowercase_       ,				self					).apply_gradients(zip(lowercase_       ,				lowercase_					)       ,				name=lowercase_       ,				**lowercase_					)


				def   A_ (		self    :					List[Any]       ,				lowercase_    :					str       ,				lowercase_    :					str       ,				lowercase_    :					Any					):
									if apply_state is None:
														return self._decayed_lr_t[var_dtype], {}

									snake_case_									=				apply_state or {}
									snake_case_									=				apply_state.get((var_device, var_dtype)					)
									if coefficients is None:
														snake_case_									=				self._fallback_apply_state(lowercase_       ,				lowercase_					)
														snake_case_									=				coefficients

									return coefficients["lr_t"], {"apply_state": apply_state}


				def   A_ (		self    :					Union[str, Any]       ,				lowercase_    :					int       ,				lowercase_    :					Union[str, Any]       ,				lowercase_    :					Optional[int]=None					):
									snake_case_			,snake_case_									=				self._get_lr(var.device       ,				var.dtype.base_dtype       ,				lowercase_					)
									snake_case_									=				self._decay_weights_op(lowercase_       ,				lowercase_       ,				lowercase_					)
									with tf.control_dependencies([decay]					):
														return super(lowercase_       ,				self					)._resource_apply_dense(lowercase_       ,				lowercase_       ,				**lowercase_					)


				def   A_ (		self    :					Optional[Any]       ,				lowercase_    :					Optional[Any]       ,				lowercase_    :					int       ,				lowercase_    :					str       ,				lowercase_    :					List[Any]=None					):
									snake_case_			,snake_case_									=				self._get_lr(var.device       ,				var.dtype.base_dtype       ,				lowercase_					)
									snake_case_									=				self._decay_weights_op(lowercase_       ,				lowercase_       ,				lowercase_					)
									with tf.control_dependencies([decay]					):
														return super(lowercase_       ,				self					)._resource_apply_sparse(lowercase_       ,				lowercase_       ,				lowercase_       ,				**lowercase_					)


				def   A_ (		self    :					Union[str, Any]					):
									snake_case_									=				super().get_config()
									config.update({'''weight_decay_rate''': self.weight_decay_rate}					)
									return config


				def   A_ (		self    :					Optional[int]       ,				lowercase_    :					int					):
									if self.weight_decay_rate == 0:
														return False

									if self._include_in_weight_decay:
														for r in self._include_in_weight_decay:
																			if re.search(lowercase_       ,				lowercase_					) is not None:
																								return True

									if self._exclude_from_weight_decay:
														for r in self._exclude_from_weight_decay:
																			if re.search(lowercase_       ,				lowercase_					) is not None:
																								return False
									return True







class 						a				(   _lowerCamelCase     ):


				def __init__(		self    :					List[Any]					):
									snake_case_									=				[]
									snake_case_									=				None


				@property
				def   A_ (		self    :					Union[str, Any]					):
									if self._accum_steps is None:
														snake_case_									=				tf.Variable(
														    tf.constant(0       ,				dtype=tf.intaa					)       ,				trainable=lowercase_       ,				synchronization=tf.VariableSynchronization.ON_READ       ,				aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA       ,				)

									return self._accum_steps.value()


				@property
				def   A_ (		self    :					Dict					):
									if not self._gradients:
														raise ValueError('''The accumulator should be called first to initialize the gradients'''					)
									return [gradient.value() if gradient is not None else gradient for gradient in self._gradients]


				def __call__(		self    :					Any       ,				lowercase_    :					int					):
									if not self._gradients:
														snake_case_									=				self.step  # Create the step variable.
														self._gradients.extend(
														    [
														        tf.Variable(
														            tf.zeros_like(lowercase_					)       ,				trainable=lowercase_       ,				synchronization=tf.VariableSynchronization.ON_READ       ,				aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA       ,				)
														        if gradient is not None
														        else gradient
														        for gradient in gradients
														    ]					)
									if len(lowercase_					) != len(self._gradients					):
														raise ValueError(F"Expected {len(self._gradients					)} gradients, but got {len(lowercase_					)}"					)

									for accum_gradient, gradient in zip(self._gradients       ,				lowercase_					):
														if accum_gradient is not None and gradient is not None:
																			accum_gradient.assign_add(lowercase_					)

									self._accum_steps.assign_add(1					)


				def   A_ (		self    :					Optional[int]					):
									if not self._gradients:
														return
									self._accum_steps.assign(0					)
									for gradient in self._gradients:
														if gradient is not None:
																			gradient.assign(tf.zeros_like(lowercase_					)					)

