






import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    MODEL_FOR_PRETRAINING_MAPPING,
		    MegatronBertForCausalLM,
		    MegatronBertForMaskedLM,
		    MegatronBertForMultipleChoice,
		    MegatronBertForNextSentencePrediction,
		    MegatronBertForPreTraining,
		    MegatronBertForQuestionAnswering,
		    MegatronBertForSequenceClassification,
		    MegatronBertForTokenClassification,
		    MegatronBertModel,
		)



class   lowerCamelCase      :







		'''simple docstring'''



		def __init__(							self				,		_UpperCamelCase				,		_UpperCamelCase=1_3				,		_UpperCamelCase=7				,		_UpperCamelCase=True				,		_UpperCamelCase=True				,		_UpperCamelCase=True				,		_UpperCamelCase=True				,		_UpperCamelCase=9_9				,		_UpperCamelCase=6_4				,		_UpperCamelCase=3_2				,		_UpperCamelCase=5				,		_UpperCamelCase=4				,		_UpperCamelCase=3_7				,		_UpperCamelCase="gelu"				,		_UpperCamelCase=0.1				,		_UpperCamelCase=0.1				,		_UpperCamelCase=5_1_2				,		_UpperCamelCase=1_6				,		_UpperCamelCase=2				,		_UpperCamelCase=0.02				,		_UpperCamelCase=3				,		_UpperCamelCase=4				,		_UpperCamelCase=None				,		)	->							Tuple:
							UpperCAmelCase_    :      Union[str, Any]            =						parent
							UpperCAmelCase_    :      Tuple            =						batch_size
							UpperCAmelCase_    :      Dict            =						seq_length
							UpperCAmelCase_    :      str            =						is_training
							UpperCAmelCase_    :      str            =						use_input_mask
							UpperCAmelCase_    :      Tuple            =						use_token_type_ids
							UpperCAmelCase_    :      Optional[Any]            =						use_labels
							UpperCAmelCase_    :      List[Any]            =						vocab_size
							UpperCAmelCase_    :      Any            =						hidden_size
							UpperCAmelCase_    :      Optional[int]            =						embedding_size
							UpperCAmelCase_    :      Dict            =						num_hidden_layers
							UpperCAmelCase_    :      Optional[Any]            =						num_attention_heads
							UpperCAmelCase_    :      Any            =						intermediate_size
							UpperCAmelCase_    :      List[Any]            =						hidden_act
							UpperCAmelCase_    :      Dict            =						hidden_dropout_prob
							UpperCAmelCase_    :      Optional[int]            =						attention_probs_dropout_prob
							UpperCAmelCase_    :      Tuple            =						max_position_embeddings
							UpperCAmelCase_    :      int            =						type_vocab_size
							UpperCAmelCase_    :      Optional[int]            =						type_sequence_label_size
							UpperCAmelCase_    :      Tuple            =						initializer_range
							UpperCAmelCase_    :      List[Any]            =						num_labels
							UpperCAmelCase_    :      str            =						num_choices
							UpperCAmelCase_    :      Optional[Any]            =						scope



		def 					__UpperCAmelCase			(							self   )	->							Optional[int]:
							UpperCAmelCase_    :      int            =						ids_tensor([self.batch_size, self.seq_length]				,		self.vocab_size   )

							UpperCAmelCase_    :      Any            =						None
							if self.use_input_mask:
												UpperCAmelCase_    :      Union[str, Any]            =						random_attention_mask([self.batch_size, self.seq_length]   )

							UpperCAmelCase_    :      Tuple            =						None
							if self.use_token_type_ids:
												UpperCAmelCase_    :      Optional[Any]            =						ids_tensor([self.batch_size, self.seq_length]				,		self.type_vocab_size   )

							UpperCAmelCase_    :      Any            =						None
							UpperCAmelCase_    :      Any            =						None
							UpperCAmelCase_    :      Any            =						None
							if self.use_labels:
												UpperCAmelCase_    :      Any            =						ids_tensor([self.batch_size]				,		self.type_sequence_label_size   )
												UpperCAmelCase_    :      int            =						ids_tensor([self.batch_size, self.seq_length]				,		self.num_labels   )
												UpperCAmelCase_    :      Any            =						ids_tensor([self.batch_size]				,		self.num_choices   )

							UpperCAmelCase_    :      List[str]            =						self.get_config()

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



		def 					__UpperCAmelCase			(							self   )	->							Optional[Any]:
							return MegatronBertConfig(
							    vocab_size=self.vocab_size				,		hidden_size=self.hidden_size				,		num_hidden_layers=self.num_hidden_layers				,		num_attention_heads=self.num_attention_heads				,		intermediate_size=self.intermediate_size				,		embedding_size=self.embedding_size				,		hidden_act=self.hidden_act				,		hidden_dropout_prob=self.hidden_dropout_prob				,		attention_probs_dropout_prob=self.attention_probs_dropout_prob				,		max_position_embeddings=self.max_position_embeddings				,		type_vocab_size=self.type_vocab_size				,		is_decoder=_UpperCamelCase				,		initializer_range=self.initializer_range				,		)



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase   )	->							Dict:
							UpperCAmelCase_    :      Dict            =						MegatronBertModel(config=_UpperCamelCase   )
							model.to(_UpperCamelCase   )
							model.eval()
							UpperCAmelCase_    :      Dict            =						model(_UpperCamelCase				,		attention_mask=_UpperCamelCase				,		token_type_ids=_UpperCamelCase   )
							UpperCAmelCase_    :      Optional[int]            =						model(_UpperCamelCase				,		token_type_ids=_UpperCamelCase   )
							UpperCAmelCase_    :      Tuple            =						model(_UpperCamelCase   )

							self.parent.assertEqual(result.last_hidden_state.shape				,		(self.batch_size, self.seq_length, self.hidden_size)   )
							self.parent.assertEqual(result.pooler_output.shape				,		(self.batch_size, self.hidden_size)   )



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase   )	->							Optional[Any]:
							UpperCAmelCase_    :      List[Any]            =						MegatronBertForMaskedLM(config=_UpperCamelCase   )
							model.to(_UpperCamelCase   )
							model.eval()
							UpperCAmelCase_    :      Tuple            =						model(_UpperCamelCase				,		attention_mask=_UpperCamelCase				,		token_type_ids=_UpperCamelCase				,		labels=_UpperCamelCase   )
							self.parent.assertEqual(result.logits.shape				,		(self.batch_size, self.seq_length, self.vocab_size)   )



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase   )	->							Optional[int]:
							UpperCAmelCase_    :      Any            =						MegatronBertForCausalLM(config=_UpperCamelCase   )
							model.to(_UpperCamelCase   )
							model.eval()
							UpperCAmelCase_    :      List[Any]            =						model(_UpperCamelCase				,		attention_mask=_UpperCamelCase				,		token_type_ids=_UpperCamelCase				,		labels=_UpperCamelCase   )
							self.parent.assertEqual(result.logits.shape				,		(self.batch_size, self.seq_length, self.vocab_size)   )



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase   )	->							str:
							UpperCAmelCase_    :      Optional[int]            =						MegatronBertForNextSentencePrediction(config=_UpperCamelCase   )
							model.to(_UpperCamelCase   )
							model.eval()
							UpperCAmelCase_    :      Optional[Any]            =						model(
							    _UpperCamelCase				,		attention_mask=_UpperCamelCase				,		token_type_ids=_UpperCamelCase				,		labels=_UpperCamelCase				,		)
							self.parent.assertEqual(result.logits.shape				,		(self.batch_size, 2)   )



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase   )	->							Optional[Any]:
							UpperCAmelCase_    :      Optional[int]            =						MegatronBertForPreTraining(config=_UpperCamelCase   )
							model.to(_UpperCamelCase   )
							model.eval()
							UpperCAmelCase_    :      List[str]            =						model(
							    _UpperCamelCase				,		attention_mask=_UpperCamelCase				,		token_type_ids=_UpperCamelCase				,		labels=_UpperCamelCase				,		next_sentence_label=_UpperCamelCase				,		)
							self.parent.assertEqual(result.prediction_logits.shape				,		(self.batch_size, self.seq_length, self.vocab_size)   )
							self.parent.assertEqual(result.seq_relationship_logits.shape				,		(self.batch_size, 2)   )



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase   )	->							Any:
							UpperCAmelCase_    :      Optional[int]            =						MegatronBertForQuestionAnswering(config=_UpperCamelCase   )
							model.to(_UpperCamelCase   )
							model.eval()
							UpperCAmelCase_    :      Dict            =						model(
							    _UpperCamelCase				,		attention_mask=_UpperCamelCase				,		token_type_ids=_UpperCamelCase				,		start_positions=_UpperCamelCase				,		end_positions=_UpperCamelCase				,		)
							self.parent.assertEqual(result.start_logits.shape				,		(self.batch_size, self.seq_length)   )
							self.parent.assertEqual(result.end_logits.shape				,		(self.batch_size, self.seq_length)   )



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase   )	->							List[Any]:
							UpperCAmelCase_    :      Union[str, Any]            =						self.num_labels
							UpperCAmelCase_    :      List[Any]            =						MegatronBertForSequenceClassification(_UpperCamelCase   )
							model.to(_UpperCamelCase   )
							model.eval()
							UpperCAmelCase_    :      Any            =						model(_UpperCamelCase				,		attention_mask=_UpperCamelCase				,		token_type_ids=_UpperCamelCase				,		labels=_UpperCamelCase   )
							self.parent.assertEqual(result.logits.shape				,		(self.batch_size, self.num_labels)   )



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase   )	->							str:
							UpperCAmelCase_    :      Union[str, Any]            =						self.num_labels
							UpperCAmelCase_    :      Optional[Any]            =						MegatronBertForTokenClassification(config=_UpperCamelCase   )
							model.to(_UpperCamelCase   )
							model.eval()
							UpperCAmelCase_    :      Optional[int]            =						model(_UpperCamelCase				,		attention_mask=_UpperCamelCase				,		token_type_ids=_UpperCamelCase				,		labels=_UpperCamelCase   )
							self.parent.assertEqual(result.logits.shape				,		(self.batch_size, self.seq_length, self.num_labels)   )



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase   )	->							Dict:
							UpperCAmelCase_    :      Optional[int]            =						self.num_choices
							UpperCAmelCase_    :      str            =						MegatronBertForMultipleChoice(config=_UpperCamelCase   )
							model.to(_UpperCamelCase   )
							model.eval()
							UpperCAmelCase_    :      Optional[Any]            =						input_ids.unsqueeze(1   ).expand(-1				,		self.num_choices				,		-1   ).contiguous()
							UpperCAmelCase_    :      List[str]            =						token_type_ids.unsqueeze(1   ).expand(-1				,		self.num_choices				,		-1   ).contiguous()
							UpperCAmelCase_    :      List[str]            =						input_mask.unsqueeze(1   ).expand(-1				,		self.num_choices				,		-1   ).contiguous()
							UpperCAmelCase_    :      Tuple            =						model(
							    _UpperCamelCase				,		attention_mask=_UpperCamelCase				,		token_type_ids=_UpperCamelCase				,		labels=_UpperCamelCase				,		)
							self.parent.assertEqual(result.logits.shape				,		(self.batch_size, self.num_choices)   )



		def 					__UpperCAmelCase			(							self   )	->							Union[str, Any]:
							UpperCAmelCase_    :      Dict            =						self.prepare_config_and_inputs()
							(
							    (
							    UpperCAmelCase_
							)       ,  (
							    UpperCAmelCase_
							)       ,  (
							    UpperCAmelCase_
							)       ,  (
							    UpperCAmelCase_
							)       ,  (
							    UpperCAmelCase_
							)       ,  (
							    UpperCAmelCase_
							)       ,  (
							    UpperCAmelCase_
							)       ,  
							)    :      Optional[int]            =						config_and_inputs
							UpperCAmelCase_    :      Optional[Any]            =						{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
							return config, inputs_dict





@require_torch
class   lowerCamelCase      (_snake_case ,  _snake_case ,  unittest.TestCase			):







		'''simple docstring'''



		_snake_case    :    Tuple       						=  (
		    (
		        MegatronBertModel,
		        MegatronBertForMaskedLM,
		        MegatronBertForCausalLM,
		        MegatronBertForMultipleChoice,
		        MegatronBertForNextSentencePrediction,
		        MegatronBertForPreTraining,
		        MegatronBertForQuestionAnswering,
		        MegatronBertForSequenceClassification,
		        MegatronBertForTokenClassification,
		    )
		    if is_torch_available()
		    else ()
		)
		_snake_case    :    str       						=  (
		    {
		        '''feature-extraction''': MegatronBertModel,
		        '''fill-mask''': MegatronBertForMaskedLM,
		        '''question-answering''': MegatronBertForQuestionAnswering,
		        '''text-classification''': MegatronBertForSequenceClassification,
		        '''text-generation''': MegatronBertForCausalLM,
		        '''token-classification''': MegatronBertForTokenClassification,
		        '''zero-shot''': MegatronBertForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)
		_snake_case    :    Any       						=  True
		# test_resize_embeddings = False
		_snake_case    :    List[str]       						=  False



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase=False   )	->							Any:
							UpperCAmelCase_    :      Dict            =						super()._prepare_for_class(_UpperCamelCase				,		_UpperCamelCase				,		return_labels=_UpperCamelCase   )

							if return_labels:
												if model_class in get_values(_UpperCamelCase   ):
																	UpperCAmelCase_    :      int            =						torch.zeros(
																	    (self.model_tester.batch_size, self.model_tester.seq_length)				,		dtype=torch.long				,		device=_UpperCamelCase   )
																	UpperCAmelCase_    :      int            =						torch.zeros(
																	    self.model_tester.batch_size				,		dtype=torch.long				,		device=_UpperCamelCase   )
							return inputs_dict



		def 					__UpperCAmelCase			(							self   )	->							Tuple:
							UpperCAmelCase_    :      int            =						MegatronBertModelTester(self   )
							UpperCAmelCase_    :      Dict            =						ConfigTester(self				,		config_class=_UpperCamelCase				,		hidden_size=3_7   )



		def 					__UpperCAmelCase			(							self   )	->							Optional[int]:
							self.config_tester.run_common_tests()



		def 					__UpperCAmelCase			(							self   )	->							Any:
							UpperCAmelCase_    :      str            =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_model(*_UpperCamelCase   )



		def 					__UpperCAmelCase			(							self   )	->							str:
							UpperCAmelCase_    :      Optional[Any]            =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_masked_lm(*_UpperCamelCase   )



		def 					__UpperCAmelCase			(							self   )	->							List[str]:
							UpperCAmelCase_    :      Dict            =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*_UpperCamelCase   )



		def 					__UpperCAmelCase			(							self   )	->							Optional[Any]:
							UpperCAmelCase_    :      str            =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*_UpperCamelCase   )



		def 					__UpperCAmelCase			(							self   )	->							Optional[int]:
							UpperCAmelCase_    :      Any            =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_pretraining(*_UpperCamelCase   )



		def 					__UpperCAmelCase			(							self   )	->							str:
							UpperCAmelCase_    :      Optional[int]            =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_question_answering(*_UpperCamelCase   )



		def 					__UpperCAmelCase			(							self   )	->							int:
							UpperCAmelCase_    :      Any            =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*_UpperCamelCase   )



		def 					__UpperCAmelCase			(							self   )	->							Optional[int]:
							UpperCAmelCase_    :      List[Any]            =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_token_classification(*_UpperCamelCase   )





def      lowercase__      (     __snake_case    :							Optional[Any]  ):


					'''simple docstring'''


					return torch.tensor(
					    __snake_case     ,       dtype=torch.long     ,       device=__snake_case     ,       )


__UpperCAmelCase								=						1E-4






@require_torch
@require_sentencepiece
@require_tokenizers
class   lowerCamelCase      (unittest.TestCase			):







		'''simple docstring'''



		@slow
		@unittest.skip('Model is not available.'   )
		def 					__UpperCAmelCase			(							self   )	->							Dict:
							UpperCAmelCase_    :      int            =						'nvidia/megatron-bert-uncased-345m'
							if "MYDIR" in os.environ:
												UpperCAmelCase_    :      List[str]            =						os.path.join(os.environ['MYDIR']				,		_UpperCamelCase   )
							UpperCAmelCase_    :      Tuple            =						MegatronBertModel.from_pretrained(_UpperCamelCase   )
							model.to(_UpperCamelCase   )
							model.half()
							UpperCAmelCase_    :      Any            =						_long_tensor([[1_0_1, 7_1_1_0, 1_0_0_5, 1_0_5_6, 2_0_2_3, 1_1_3_3_3, 1_7_4_1_3, 1_0_2_9, 1_0_2]]   )
							with torch.no_grad():
												UpperCAmelCase_    :      str            =						model(_UpperCamelCase   )[0]
							UpperCAmelCase_    :      List[Any]            =						torch.Size((1, 9, 1_0_2_4)   )
							self.assertEqual(output.shape				,		_UpperCamelCase   )

							UpperCAmelCase_    :      Optional[int]            =						[-0.60_40, -0.25_17, -0.10_25, 0.34_20, -0.67_58, -0.00_17, -0.10_89, -0.19_90, 0.57_28]
							for ii in range(3   ):
												for jj in range(3   ):
																	UpperCAmelCase_    :      Optional[int]            =						output[0, ii, jj]
																	UpperCAmelCase_    :      Any            =						expected[3 * ii + jj]
																	UpperCAmelCase_    :      List[Any]            =						'ii={} jj={} a={} b={}'.format(_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase   )
																	self.assertTrue(math.isclose(_UpperCamelCase				,		_UpperCamelCase				,		rel_tol=_UpperCamelCase				,		abs_tol=_UpperCamelCase   )				,		msg=_UpperCamelCase   )










"""simple docstring"""







import math
import random




def    UpperCAmelCase__       (lowerCAmelCase_  ,						lowerCAmelCase_ = False   ):




					'''simple docstring'''


					if deriv:
										return value * (1 - value)
					return 1 / (1 + math.exp(-value   ))


# Initial Value
a__				:							Tuple						=     0.02




def    UpperCAmelCase__       (lowerCAmelCase_  ,						lowerCAmelCase_   ):




					'''simple docstring'''


					__SCREAMING_SNAKE_CASE						   =				float(2 * (random.randint(1  ,						100   )) - 1   )

					for _ in range(lowerCAmelCase_   ):
										# Forward propagation
										__SCREAMING_SNAKE_CASE						   =				sigmoid_function(INITIAL_VALUE * weight   )
										# How much did we miss?
										__SCREAMING_SNAKE_CASE						   =				(expected / 100) - layer_a
										# Error delta
										__SCREAMING_SNAKE_CASE						   =				layer_1_error * sigmoid_function(lowerCAmelCase_  ,						lowerCAmelCase_   )
										# Update weight
										weight += INITIAL_VALUE * layer_1_delta

					return layer_a * 100


if __name__ == "__main__":
		import doctest

		doctest.testmod()

		a__				:							List[str]						=     int(input('''Expected value: '''))
		a__				:							str						=     int(input('''Number of propagations: '''))
		print(forward_propagation(expected, number_propagations))
