


"""simple docstring"""







import unittest

import numpy as np

from transformers import DistilBertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
		import jax.numpy as jnp

		from transformers.models.distilbert.modeling_flax_distilbert import (
		    FlaxDistilBertForMaskedLM,
		    FlaxDistilBertForMultipleChoice,
		    FlaxDistilBertForQuestionAnswering,
		    FlaxDistilBertForSequenceClassification,
		    FlaxDistilBertForTokenClassification,
		    FlaxDistilBertModel,
		)




class  UpperCamelCase_	( unittest.TestCase):

	"""simple docstring"""





	def __init__( self			:				Tuple			,			UpperCAmelCase__			:				Any			,			UpperCAmelCase__			:				List[str]=1_3			,			UpperCAmelCase__			:				Optional[int]=7			,			UpperCAmelCase__			:				Any=True			,			UpperCAmelCase__			:				Dict=True			,			UpperCAmelCase__			:				int=True			,			UpperCAmelCase__			:				List[str]=True			,			UpperCAmelCase__			:				Union[str, Any]=9_9			,			UpperCAmelCase__			:				Any=3_2			,			UpperCAmelCase__			:				Tuple=5			,			UpperCAmelCase__			:				Optional[Any]=4			,			UpperCAmelCase__			:				List[Any]=3_7			,			UpperCAmelCase__			:				Any="gelu"			,			UpperCAmelCase__			:				List[Any]=0.1			,			UpperCAmelCase__			:				Union[str, Any]=0.1			,			UpperCAmelCase__			:				Union[str, Any]=5_1_2			,			UpperCAmelCase__			:				Union[str, Any]=1_6			,			UpperCAmelCase__			:				List[str]=2			,			UpperCAmelCase__			:				List[Any]=0.02			,			UpperCAmelCase__			:				List[str]=4			,			)       ->	int:
						__SCREAMING_SNAKE_CASE						   =				parent
						__SCREAMING_SNAKE_CASE						   =				batch_size
						__SCREAMING_SNAKE_CASE						   =				seq_length
						__SCREAMING_SNAKE_CASE						   =				is_training
						__SCREAMING_SNAKE_CASE						   =				use_attention_mask
						__SCREAMING_SNAKE_CASE						   =				use_token_type_ids
						__SCREAMING_SNAKE_CASE						   =				use_labels
						__SCREAMING_SNAKE_CASE						   =				vocab_size
						__SCREAMING_SNAKE_CASE						   =				hidden_size
						__SCREAMING_SNAKE_CASE						   =				num_hidden_layers
						__SCREAMING_SNAKE_CASE						   =				num_attention_heads
						__SCREAMING_SNAKE_CASE						   =				intermediate_size
						__SCREAMING_SNAKE_CASE						   =				hidden_act
						__SCREAMING_SNAKE_CASE						   =				hidden_dropout_prob
						__SCREAMING_SNAKE_CASE						   =				attention_probs_dropout_prob
						__SCREAMING_SNAKE_CASE						   =				max_position_embeddings
						__SCREAMING_SNAKE_CASE						   =				type_vocab_size
						__SCREAMING_SNAKE_CASE						   =				type_sequence_label_size
						__SCREAMING_SNAKE_CASE						   =				initializer_range
						__SCREAMING_SNAKE_CASE						   =				num_choices




	def   UpperCAmelCase_      ( self			:				str   )       ->	Tuple:
						__SCREAMING_SNAKE_CASE						   =				ids_tensor([self.batch_size, self.seq_length]			,			self.vocab_size   )

						__SCREAMING_SNAKE_CASE						   =				None
						if self.use_attention_mask:
											__SCREAMING_SNAKE_CASE						   =				random_attention_mask([self.batch_size, self.seq_length]   )

						__SCREAMING_SNAKE_CASE						   =				DistilBertConfig(
						    vocab_size=self.vocab_size			,			dim=self.hidden_size			,			n_layers=self.num_hidden_layers			,			n_heads=self.num_attention_heads			,			hidden_dim=self.intermediate_size			,			hidden_act=self.hidden_act			,			dropout=self.hidden_dropout_prob			,			attention_dropout=self.attention_probs_dropout_prob			,			max_position_embeddings=self.max_position_embeddings			,			initializer_range=self.initializer_range			,			tie_weights_=__a			,			)

						return config, input_ids, attention_mask




	def   UpperCAmelCase_      ( self			:				Any   )       ->	Dict:
						__SCREAMING_SNAKE_CASE						   =				self.prepare_config_and_inputs()
						__SCREAMING_SNAKE_CASE						   =				config_and_inputs
						__SCREAMING_SNAKE_CASE						   =				{"input_ids": input_ids, "attention_mask": attention_mask}
						return config, inputs_dict




@require_flax
class  UpperCamelCase_	( UpperCamelCase			,  unittest.TestCase):

	"""simple docstring"""





	snake_case__				:  Dict    					=			(
	    (
	        FlaxDistilBertModel,
	        FlaxDistilBertForMaskedLM,
	        FlaxDistilBertForMultipleChoice,
	        FlaxDistilBertForQuestionAnswering,
	        FlaxDistilBertForSequenceClassification,
	        FlaxDistilBertForTokenClassification,
	        FlaxDistilBertForQuestionAnswering,
	    )
	    if is_flax_available()
	    else ()
	)




	def   UpperCAmelCase_      ( self			:				List[str]   )       ->	Dict:
						__SCREAMING_SNAKE_CASE						   =				FlaxDistilBertModelTester(self   )




	@slow
	def   UpperCAmelCase_      ( self			:				Any   )       ->	List[str]:

						for model_class_name in self.all_model_classes:
											__SCREAMING_SNAKE_CASE						   =				model_class_name.from_pretrained("distilbert-base-uncased"   )
											__SCREAMING_SNAKE_CASE						   =				model(np.ones((1, 1)   )   )
											self.assertIsNotNone(__a   )




@require_flax
class  UpperCamelCase_	( unittest.TestCase):

	"""simple docstring"""





	@slow
	def   UpperCAmelCase_      ( self			:				Optional[Any]   )       ->	Optional[int]:
						__SCREAMING_SNAKE_CASE						   =				FlaxDistilBertModel.from_pretrained("distilbert-base-uncased"   )
						__SCREAMING_SNAKE_CASE						   =				np.array([[0, 3_4_5, 2_3_2, 3_2_8, 7_4_0, 1_4_0, 1_6_9_5, 6_9, 6_0_7_8, 1_5_8_8, 2]]   )
						__SCREAMING_SNAKE_CASE						   =				np.array([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]   )
						__SCREAMING_SNAKE_CASE						   =				model(__a			,			attention_mask=__a   )[0]
						__SCREAMING_SNAKE_CASE						   =				(1, 1_1, 7_6_8)
						self.assertEqual(output.shape			,			__a   )
						__SCREAMING_SNAKE_CASE						   =				np.array([[[-0.1_639, 0.3_299, 0.1_648], [-0.1_746, 0.3_289, 0.1_710], [-0.1_884, 0.3_357, 0.1_810]]]   )

						self.assertTrue(jnp.allclose(output[:, 1:4, 1:4]			,			__a			,			atol=1E-4   )   )





from typing import Dict

from transformers import EvalPrediction, HfArgumentParser, TrainingArguments, is_torch_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    get_torch_dist_unique_port,
    require_torch_multi_gpu,
    require_torch_neuroncore,
)
from transformers.training_args import ParallelMode
from transformers.utils import logging


_snake_case		=  logging.get_logger(__name__)


if is_torch_available():
						import torch
						from torch import nn
						from torch.utils.data import Dataset

						from transformers import Trainer



						class   UpperCAmelCase_    (       a):







													def __init__(   self,			__a = 101):

																				'''simple docstring'''

																				_lowerCAmelCase     :							str    =    length







													def __len__(   self):

																				'''simple docstring'''

																				return self.length







													def __getitem__(   self,			__a):

																				'''simple docstring'''

																				return i



						class   UpperCAmelCase_    :







													def __call__(   self,			__a):

																				'''simple docstring'''

																				return {"input_ids": torch.tensor(__a), "labels": torch.tensor(__a)}



						class   UpperCAmelCase_    (       nn.Module):







													def __init__(   self):

																				'''simple docstring'''

																				super().__init__()
																				# Add some (unused) params otherwise DDP will complain.
																				_lowerCAmelCase     :							str    =    nn.Linear(120,			80)







													def    snake_case__      (   self,			__a,			__a=None):

																				'''simple docstring'''

																				if labels is not None:
																											return torch.tensor(0.0,			device=input_ids.device), input_ids
																				else:
																											return input_ids



class   UpperCAmelCase_    (       a):







							@require_torch_neuroncore
							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							int    =    f"--nproc_per_node=2\n            --master_port={get_torch_dist_unique_port()}\n            {self.test_file_dir}/test_trainer_distributed.py\n        ".split()
														_lowerCAmelCase     :							Tuple    =    self.get_auto_remove_tmp_dir()
														_lowerCAmelCase     :							Optional[int]    =    f"--output_dir {output_dir}".split()
														_lowerCAmelCase     :							List[Any]    =    ["torchrun"] + distributed_args + args
														execute_subprocess_async(__a,			env=self.get_env())
														# successful return here == success - any errors would have caused an error in the sub-call



class   UpperCAmelCase_    (       a):







							@require_torch_multi_gpu
							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Dict    =    f"--nproc_per_node={torch.cuda.device_count()}\n            --master_port={get_torch_dist_unique_port()}\n            {self.test_file_dir}/test_trainer_distributed.py\n        ".split()
														_lowerCAmelCase     :							Any    =    self.get_auto_remove_tmp_dir()
														_lowerCAmelCase     :							Optional[int]    =    f"--output_dir {output_dir}".split()
														_lowerCAmelCase     :							Any    =    ["torchrun"] + distributed_args + args
														execute_subprocess_async(__a,			env=self.get_env())
														# successful return here == success - any errors would have caused an error in the sub-call


if __name__ == "__main__":
						# The script below is meant to be run under torch.distributed, on a machine with multiple GPUs:
						#
						# PYTHONPATH="src" python -m torch.distributed.run --nproc_per_node 2 --output_dir output_dir ./tests/test_trainer_distributed.py

						_snake_case		=  HfArgumentParser((TrainingArguments,))
						_snake_case		=  parser.parse_args_into_dataclasses()[0]

						logger.warning(
						    f'''Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, '''
						    f'''distributed training: {training_args.parallel_mode != ParallelMode.NOT_DISTRIBUTED}'''
						)

						# Essentially, what we want to verify in the distributed case is that we get all samples back,
						# in the right order. (this is crucial for prediction for instance)
						for dataset_length in [101, 40, 7]:
												_snake_case		=  DummyDataset(dataset_length)






												def 				A				(					_lowerCamelCase  ):





																			'''simple docstring'''




																			_lowerCAmelCase     :							Dict    =    list(range(len(_lowerCamelCase  )  )  )
																			_lowerCAmelCase     :							Union[str, Any]    =    p.predictions.tolist() == sequential and p.label_ids.tolist() == sequential
																			if not success and training_args.local_rank == 0:
																										logger.warning(
																										    "Predictions and/or labels do not match expected results:\n  - predictions: "
																										    F"{p.predictions.tolist()}\n  - labels: {p.label_ids.tolist()}\n  - expected: {sequential}"  )
																			return {"success": success}

												_snake_case		=  Trainer(
												    model=DummyModel(),
												    args=training_args,
												    data_collator=DummyDataCollator(),
												    eval_dataset=dataset,
												    compute_metrics=compute_metrics,
												)
												_snake_case		=  trainer.evaluate()
												logger.info(metrics)
												if metrics["eval_success"] is not True:
																		logger.error(metrics)
																		exit(1)

												_snake_case		=  trainer.predict(dataset)
												logger.info(p.metrics)
												if p.metrics["test_success"] is not True:
																		logger.error(p.metrics)
																		exit(1)

												_snake_case		=  2

												_snake_case		=  trainer.evaluate()
												logger.info(metrics)
												if metrics["eval_success"] is not True:
																		logger.error(metrics)
																		exit(1)

												_snake_case		=  trainer.predict(dataset)
												logger.info(p.metrics)
												if p.metrics["test_success"] is not True:
																		logger.error(p.metrics)
																		exit(1)



												_snake_case		=  None
