


import itertools
import os
import subprocess
from os.path import dirname

from parameterized import parameterized

from tests.trainer.test_trainer import TrainerIntegrationCommon  # noqa
from transformers import is_torch_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    get_gpu_count,
    get_tests_dir,
    require_deepspeed,
    require_torch_gpu,
    slow,
)
from transformers.trainer_utils import set_seed


if is_torch_available():
					from tests.trainer.test_trainer import (  # noqa
					    RegressionModelConfig,
					    RegressionPreTrainedModel,
					    get_regression_trainer,
					)


set_seed(42)

UpperCamelCase			:					Dict         =							get_tests_dir('''fixtures''')
UpperCamelCase			:					Optional[Any]         =							os.path.join(dirname(get_tests_dir()))
UpperCamelCase			:					List[str]         =							dirname(os.path.abspath(__file__))

# default torch.distributed port
UpperCamelCase			:					Tuple         =							'''10999'''

UpperCamelCase			:					str         =							'''t5-small'''

# *** Working Models ***
UpperCamelCase			:					Any         =							'''hf-internal-testing/tiny-albert'''
UpperCamelCase			:					Union[str, Any]         =							'''sshleifer/bart-tiny-random'''
UpperCamelCase			:					Union[str, Any]         =							'''hf-internal-testing/tiny-bert'''
UpperCamelCase			:					int         =							'''hf-internal-testing/tiny-random-bigbird_pegasus'''
UpperCamelCase			:					Union[str, Any]         =							'''hf-internal-testing/tiny-random-big_bird'''
UpperCamelCase			:					List[str]         =							'''hf-internal-testing/tiny-random-blenderbot'''
UpperCamelCase			:					List[Any]         =							'''bigscience/bigscience-small-testing'''
UpperCamelCase			:					int         =							'''hf-internal-testing/tiny-random-deberta'''
UpperCamelCase			:					Optional[Any]         =							'''hf-internal-testing/tiny-random-deberta-v2'''
UpperCamelCase			:					Dict         =							'''sshleifer/tiny-distilbert-base-cased'''
UpperCamelCase			:					Dict         =							'''hf-internal-testing/tiny-electra'''
UpperCamelCase			:					Union[str, Any]         =							'''hf-internal-testing/tiny-random-flaubert'''
UpperCamelCase			:					int         =							'''stas/tiny-wmt19-en-de'''
UpperCamelCase			:					Union[str, Any]         =							'''hf-internal-testing/tiny-random-funnel'''
UpperCamelCase			:					Optional[Any]         =							'''sshleifer/tiny-gpt2'''
UpperCamelCase			:					Tuple         =							'''hf-internal-testing/tiny-random-gptj'''
UpperCamelCase			:					Optional[int]         =							'''hf-internal-testing/tiny-random-gpt_neo'''
UpperCamelCase			:					int         =							'''hf-internal-testing/tiny-layoutlm'''
UpperCamelCase			:					List[Any]         =							'''hf-internal-testing/tiny-random-led'''
UpperCamelCase			:					Tuple         =							'''hf-internal-testing/tiny-random-longformer'''
UpperCamelCase			:					List[str]         =							'''stas/tiny-m2m_100'''  # hf tiny model is unsuitable
UpperCamelCase			:					List[Any]         =							'''sshleifer/tiny-marian-en-de'''
UpperCamelCase			:					int         =							'''sshleifer/tiny-mbart'''
UpperCamelCase			:					Union[str, Any]         =							'''hf-internal-testing/tiny-random-mobilebert'''
UpperCamelCase			:					Union[str, Any]         =							'''hf-internal-testing/tiny-random-mpnet'''
UpperCamelCase			:					List[Any]         =							'''stas/pegasus-cnn_dailymail-tiny-random'''
UpperCamelCase			:					str         =							'''hf-internal-testing/tiny-random-prophetnet'''
UpperCamelCase			:					Union[str, Any]         =							'''sshleifer/tiny-distilroberta-base'''
UpperCamelCase			:					Optional[int]         =							'''hf-internal-testing/tiny-random-squeezebert'''
UpperCamelCase			:					Tuple         =							'''patrickvonplaten/t5-tiny-random'''
UpperCamelCase			:					Tuple         =							'''hf-internal-testing/tiny-random-t5-v1.1'''
UpperCamelCase			:					Any         =							'''hf-internal-testing/tiny-random-vit'''
UpperCamelCase			:					Tuple         =							'''hf-internal-testing/tiny-xlm-roberta'''
UpperCamelCase			:					Union[str, Any]         =							'''sshleifer/tiny-xlnet-base-cased'''


# *** To Fix ***


# *** tiny model issues ***
# missing model files:
UpperCamelCase			:					int         =							'''hf-internal-testing/tiny-random-mt5'''
UpperCamelCase			:					Optional[Any]         =							'''hf-internal-testing/tiny-random-camembert'''
UpperCamelCase			:					int         =							'''hf-internal-testing/tiny-random-openai-gpt'''

# missing tokenizer files
UpperCamelCase			:					Union[str, Any]         =							'''hf-internal-testing/tiny-random-convbert'''
UpperCamelCase			:					Dict         =							'''hf-internal-testing/tiny-random-layoutlmv2'''
UpperCamelCase			:					List[str]         =							'''hf-internal-testing/tiny-random-hubert'''

# issues with tokenizer
UpperCamelCase			:					Tuple         =							'''hf-internal-testing/tiny-random-ctrl'''
UpperCamelCase			:					Optional[int]         =							'''hf-internal-testing/tiny-random-transfo-xl'''  # same as ctrl

# other issues with tiny models
UpperCamelCase			:					List[Any]         =							'''hf-internal-testing/tiny-random-ibert'''  # multiple issues with either mlm/qa/clas
UpperCamelCase			:					int         =							'''hf-internal-testing/tiny-random-reformer'''  # multiple issues with either mlm/qa/clas

# *** Lacking official examples to test with ***
# or not working with examples
UpperCamelCase			:					Tuple         =							'''hf-internal-testing/tiny-random-dpr'''
# - "dpr"  examples/research_projects/rag-end2end-retriever/
UpperCamelCase			:					Any         =							'''hf-internal-testing/tiny-random-rag'''
# - "rag" research_projects
UpperCamelCase			:					int         =							''''''
# - "luke" Entities classes - no plan to make such example
UpperCamelCase			:					List[str]         =							'''hf-internal-testing/tiny-random-lxmert'''
# - "lxmert" doesn't work with run_qa.py
UpperCamelCase			:					List[str]         =							'''hf-internal-testing/tiny-random-clip'''
# - "clip" nothing under pytorch examples - XXX: Suraj is working on adding some - check by end of Sep
UpperCamelCase			:					str         =							'''hf-internal-testing/tiny-random-speech_to_text'''
# - "speech_to_text", nothing under pytorch examples


# *** Reactive mode ***
# models with low usage, unstable API, things about to change - do nothing about the following until someone runs into a problem
UpperCamelCase			:					Optional[Any]         =							'''hf-internal-testing/tiny-random-tapas'''
def 				UpperCamelCase_	(					__snake_case      :    int=False  )->   int:




						'''simple docstring'''
						# 1. explicitly set --num_nodes=1 just in case these tests end up run on a multi-node setup
						# - it won't be able to handle that
						# 2. for now testing with just 2 gpus max (since some quality tests may give different
						# results with mode gpus because we use very little data)
						__magic_name__					=      min(2						,      get_gpu_count()  ) if distributed else 1
						__magic_name__					=      os.environ.get('DS_TEST_PORT'						,      __snake_case  )
						return F'''deepspeed --num_nodes 1 --num_gpus {num_gpus} --master_port {master_port}'''.split()
def 				UpperCamelCase_	(					)->   Union[str, Any]:




						'''simple docstring'''
						__magic_name__					=      F'''{FIXTURE_DIRECTORY}/tests_samples'''
						__magic_name__					=      F'''{data_dir_samples}/wmt_en_ro'''
						__magic_name__					=      F'''{data_dir_samples}/xsum'''
						__magic_name__					=      '\n        --do_train\n        --max_train_samples 4\n        --per_device_train_batch_size 2\n        --num_train_epochs 1\n        --fp16\n        --report_to none\n        --overwrite_output_dir\n        '.split()

						# try to cover as many models as possible once (it's enough to run on one task per model)
						# but need a tiny model for each
						#
						# should have "{model_type.upper()}_TINY" corresponding vars defined, e.g., T5_TINY, etc.
						__magic_name__					=      {
						    'trans': [
						        'bart',
						        'fsmt',
						        'm2m_100',
						        'marian',
						        'mbart',
						        't5',
						        't5_v1',
						        # "mt5", missing model files
						    ],
						    'sum': [
						        'pegasus',
						    ],
						    'clm': [
						        'big_bird',
						        'bigbird_pegasus',
						        'blenderbot',
						        'bloom',
						        'gpt2',
						        'gpt_neo',
						        'gptj',
						        'xlm-roberta',
						        'prophetnet',
						        # "camembert", missing model files
						    ],
						    'mlm': [
						        'albert',
						        'deberta',
						        'deberta-v2',
						        'distilbert',
						        'electra',
						        'flaubert',
						        'funnel',
						        'layoutlm',
						        # "reformer", # multiple issues with either mlm/qa/clas
						    ],
						    'qa': [
						        'led',
						        'longformer',
						        'mobilebert',
						        'mpnet',
						        'roberta',
						        'squeezebert',
						        # "convbert", # missing tokenizer files
						        # "layoutlmv2", missing model files
						    ],
						    'clas': [
						        'bert',
						        'xlnet',
						        # "hubert", # missing tokenizer files
						        # "ibert", # multiple issues with either mlm/qa/clas
						        # "transfo-xl", # tokenizer issues as ctrl
						        # "ctrl", # tokenizer issues
						        # "openai-gpt", missing model files
						        # "tapas", multiple issues
						    ],
						    'img_clas': [
						        'vit',
						    ],
						}

						__magic_name__					=      F'''{ROOT_DIRECTORY}/examples/pytorch'''

						__magic_name__					=      {
						    'trans': F'''
        {scripts_dir}/translation/run_translation.py
        --train_file {data_dir_wmt}/train.json
        --source_lang en
        --target_lang ro
        ''',
						    'sum': F'''
        {scripts_dir}/summarization/run_summarization.py
        --train_file {data_dir_xsum}/sample.json
        --max_source_length 12
        --max_target_length 12
        --lang en
        ''',
						    'clm': F'''
        {scripts_dir}/language-modeling/run_clm.py
        --train_file {FIXTURE_DIRECTORY}/sample_text.txt
        --block_size 8
        ''',
						    'mlm': F'''
        {scripts_dir}/language-modeling/run_mlm.py
        --train_file {FIXTURE_DIRECTORY}/sample_text.txt
        ''',
						    'qa': F'''
        {scripts_dir}/question-answering/run_qa.py
        --train_file {data_dir_samples}/SQUAD/sample.json
        ''',
						    'clas': F'''
        {scripts_dir}/text-classification/run_glue.py
        --train_file {data_dir_samples}/MRPC/train.csv
        --max_seq_length 12
        --task_name MRPC
        ''',
						    'img_clas': F'''
        {scripts_dir}/image-classification/run_image_classification.py
            --dataset_name hf-internal-testing/cats_vs_dogs_sample
            --remove_unused_columns False
            --max_steps 10
            --image_processor_name {DS_TESTS_DIRECTORY}/vit_feature_extractor.json
        ''',
						}

						__magic_name__					=      get_launcher(distributed=__snake_case  )

						__magic_name__					=      {}
						for task, args in tasks.items():
												__magic_name__					=      args.split()
												for model in tasksamodels[task]:
																		__magic_name__					=      globals()[F'''{model.upper().replace("-"						,      "_"  )}_TINY''']
																		__magic_name__					=      F'''--model_name_or_path {model_name}'''.split()
																		__magic_name__					=      launcher + args + args_model + args_main

																		# # generation special case
																		# if task == "gen":
																		#     launcher = f"deepspeed --num_nodes 1 --num_gpus 1".split()
																		#     args_model += f"--model_type {model}".split()
																		#     cmds[f"{task}_{model}"] = launcher + args + args_model
																		# else:

						return cmds


UpperCamelCase			:					Dict         =							make_task_cmds()

UpperCamelCase			:					str         =							'''zero2'''
UpperCamelCase			:					Tuple         =							'''zero3'''

UpperCamelCase			:					Optional[int]         =							[ZEROa, ZEROa]
def 				UpperCamelCase_	(					__snake_case      :    Union[str, Any]						,      __snake_case      :    List[Any]						,      __snake_case      :    Dict  )->   Dict:




						'''simple docstring'''
						# customize the test name generator function as we want both params to appear in the sub-test
						# name, as by default it shows only the first param
						__magic_name__					=      parameterized.to_safe_name('_'.join(str(__snake_case  ) for x in param.args  )  )
						return F'''{func.__name__}_{param_based_name}'''


# Cartesian-product of zero stages with models to test
UpperCamelCase			:					int         =							list(itertools.product(stages, task_cmds.keys()))




@slow
@require_deepspeed
@require_torch_gpu
class      __snake_case				(   _UpperCAmelCase):
					def 	a			(						self       :						List[str]	,      _lowerCAmelCase       :						Dict	,      _lowerCAmelCase       :						List[str]  )							->					List[Any]:



											'''simple docstring'''





											if task not in task_cmds:
																	raise ValueError(F'''don\'t know of task {task}, have {task_cmds.keys()}'''  )

											__magic_name__					=      task_cmds[task]
											__magic_name__					=      F'''--deepspeed {self.test_file_dir_str}/ds_config_{stage}.json'''.split()

											__magic_name__					=      self.get_auto_remove_tmp_dir()
											__magic_name__					=      F'''--output_dir {output_dir}'''.split()

											cmd += args_ds + args_out

											return cmd, output_dir







					@parameterized.expand(_lowerCAmelCase	,      name_func=_lowerCAmelCase  )
					def 	a			(						self       :						Tuple	,      _lowerCAmelCase       :						List[Any]	,      _lowerCAmelCase       :						Tuple  )							->					Any:



											'''simple docstring'''





											__magic_name__		,      __magic_name__					=      self.get_task_cmd(_lowerCAmelCase	,      _lowerCAmelCase  )

											# 1. generate the checkpoint
											cmd += "--save_steps 1".split()
											# keep for quick debug
											# print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] + cmd)); die
											execute_subprocess_async(_lowerCAmelCase	,      env=self.get_env()  )

											# 2. test that the fp32 weights get reconsolidated
											__magic_name__					=      F'''{output_dir}/checkpoint-1'''
											__magic_name__					=      F'''{chkpt_dir}/out.bin'''
											__magic_name__					=      F'''{chkpt_dir}/zero_to_fp32.py {chkpt_dir} {recovered_model_path}'''
											# keep for quick debug
											# print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
											subprocess.check_call(_lowerCAmelCase	,      shell=_lowerCAmelCase  )
											assert os.path.exists(_lowerCAmelCase  ), F'''{recovered_model_path} was not found'''

											# possibly could also test that the resulting saved model is usable but given that we use
											# random models we won't know if it's any good





from collections import OrderedDict
from dataclasses import fields
from typing import Any, Tuple

import numpy as np

from .import_utils import is_torch_available
def 				UpperCamelCase_	(					__snake_case      :    Dict  )->   str:




						'''simple docstring'''
						if is_torch_available():
												import torch

												if isinstance(__snake_case						,      torch.Tensor  ):
																		return True

						return isinstance(__snake_case						,      np.ndarray  )




class      __snake_case				(   _UpperCAmelCase):
					def 	a			(						self       :						Optional[Any]  )							->					Optional[Any]:



											'''simple docstring'''





											__magic_name__					=      fields(self  )

											# Safety and consistency checks
											if not len(_lowerCAmelCase  ):
																	raise ValueError(F'''{self.__class__.__name__} has no fields.'''  )

											__magic_name__					=      getattr(self	,      class_fields[0].name  )
											__magic_name__					=      all(getattr(self	,      field.name  ) is None for field in class_fields[1:]  )

											if other_fields_are_none and isinstance(_lowerCAmelCase	,      _lowerCAmelCase  ):
																	for key, value in first_field.items():
																							__magic_name__					=      value
											else:
																	for field in class_fields:
																							__magic_name__					=      getattr(self	,      field.name  )
																							if v is not None:
																													__magic_name__					=      v
					def __delitem__(						self       :						Dict	,      *_lowerCAmelCase       :						List[str]	,      **_lowerCAmelCase       :						List[str]  )							->					List[str]:



											'''simple docstring'''





											raise Exception(F'''You cannot use ``__delitem__`` on a {self.__class__.__name__} instance.'''  )
					def 	a			(						self       :						Optional[int]	,      *_lowerCAmelCase       :						Optional[Any]	,      **_lowerCAmelCase       :						Optional[Any]  )							->					str:



											'''simple docstring'''





											raise Exception(F'''You cannot use ``setdefault`` on a {self.__class__.__name__} instance.'''  )
					def 	a			(						self       :						Union[str, Any]	,      *_lowerCAmelCase       :						List[str]	,      **_lowerCAmelCase       :						Any  )							->					str:



											'''simple docstring'''





											raise Exception(F'''You cannot use ``pop`` on a {self.__class__.__name__} instance.'''  )
					def 	a			(						self       :						Any	,      *_lowerCAmelCase       :						str	,      **_lowerCAmelCase       :						Optional[int]  )							->					List[Any]:



											'''simple docstring'''





											raise Exception(F'''You cannot use ``update`` on a {self.__class__.__name__} instance.'''  )
					def __getitem__(						self       :						Optional[Any]	,      _lowerCAmelCase       :						Any  )							->					Tuple:



											'''simple docstring'''





											if isinstance(_lowerCAmelCase	,      _lowerCAmelCase  ):
																	__magic_name__					=      dict(self.items()  )
																	return inner_dict[k]
											else:
																	return self.to_tuple()[k]
					def __setattr__(						self       :						str	,      _lowerCAmelCase       :						Optional[Any]	,      _lowerCAmelCase       :						Optional[int]  )							->					int:



											'''simple docstring'''





											if name in self.keys() and value is not None:
																	# Don't call self.__setitem__ to avoid recursion errors
																	super().__setitem__(_lowerCAmelCase	,      _lowerCAmelCase  )
											super().__setattr__(_lowerCAmelCase	,      _lowerCAmelCase  )
					def __setitem__(						self       :						Optional[Any]	,      _lowerCAmelCase       :						Optional[Any]	,      _lowerCAmelCase       :						List[str]  )							->					Optional[int]:



											'''simple docstring'''





											super().__setitem__(_lowerCAmelCase	,      _lowerCAmelCase  )
											# Don't call self.__setattr__ to avoid recursion errors
											super().__setattr__(_lowerCAmelCase	,      _lowerCAmelCase  )







					def 	a			(						self       :						str  )							->					Tuple[Any]:



											'''simple docstring'''





											return tuple(self[k] for k in self.keys()  )

