






'''simple docstring'''







import itertools
import os
import subprocess
from os.path import dirname

from parameterized import parameterized

from tests.trainer.test_trainer import TrainerIntegrationCommon  # noqa
from transformers import is_torch_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    get_gpu_count,
    get_tests_dir,
    require_deepspeed,
    require_torch_gpu,
    slow,
)
from transformers.trainer_utils import set_seed


if is_torch_available():
  from tests.trainer.test_trainer import (  # noqa
      RegressionModelConfig,
      RegressionPreTrainedModel,
      get_regression_trainer,
  )


set_seed(42)

UpperCamelCase_					=				get_tests_dir("fixtures")
UpperCamelCase_					=				os.path.join(dirname(get_tests_dir()))
UpperCamelCase_					=				dirname(os.path.abspath(__file__))

# default torch.distributed port
UpperCamelCase_					=				"10999"

UpperCamelCase_					=				"t5-small"

# *** Working Models ***
UpperCamelCase_					=				"hf-internal-testing/tiny-albert"
UpperCamelCase_					=				"sshleifer/bart-tiny-random"
UpperCamelCase_					=				"hf-internal-testing/tiny-bert"
UpperCamelCase_					=				"hf-internal-testing/tiny-random-bigbird_pegasus"
UpperCamelCase_					=				"hf-internal-testing/tiny-random-big_bird"
UpperCamelCase_					=				"hf-internal-testing/tiny-random-blenderbot"
UpperCamelCase_					=				"bigscience/bigscience-small-testing"
UpperCamelCase_					=				"hf-internal-testing/tiny-random-deberta"
UpperCamelCase_					=				"hf-internal-testing/tiny-random-deberta-v2"
UpperCamelCase_					=				"sshleifer/tiny-distilbert-base-cased"
UpperCamelCase_					=				"hf-internal-testing/tiny-electra"
UpperCamelCase_					=				"hf-internal-testing/tiny-random-flaubert"
UpperCamelCase_					=				"stas/tiny-wmt19-en-de"
UpperCamelCase_					=				"hf-internal-testing/tiny-random-funnel"
UpperCamelCase_					=				"sshleifer/tiny-gpt2"
UpperCamelCase_					=				"hf-internal-testing/tiny-random-gptj"
UpperCamelCase_					=				"hf-internal-testing/tiny-random-gpt_neo"
UpperCamelCase_					=				"hf-internal-testing/tiny-layoutlm"
UpperCamelCase_					=				"hf-internal-testing/tiny-random-led"
UpperCamelCase_					=				"hf-internal-testing/tiny-random-longformer"
UpperCamelCase_					=				"stas/tiny-m2m_100"  # hf tiny model is unsuitable
UpperCamelCase_					=				"sshleifer/tiny-marian-en-de"
UpperCamelCase_					=				"sshleifer/tiny-mbart"
UpperCamelCase_					=				"hf-internal-testing/tiny-random-mobilebert"
UpperCamelCase_					=				"hf-internal-testing/tiny-random-mpnet"
UpperCamelCase_					=				"stas/pegasus-cnn_dailymail-tiny-random"
UpperCamelCase_					=				"hf-internal-testing/tiny-random-prophetnet"
UpperCamelCase_					=				"sshleifer/tiny-distilroberta-base"
UpperCamelCase_					=				"hf-internal-testing/tiny-random-squeezebert"
UpperCamelCase_					=				"patrickvonplaten/t5-tiny-random"
UpperCamelCase_					=				"hf-internal-testing/tiny-random-t5-v1.1"
UpperCamelCase_					=				"hf-internal-testing/tiny-random-vit"
UpperCamelCase_					=				"hf-internal-testing/tiny-xlm-roberta"
UpperCamelCase_					=				"sshleifer/tiny-xlnet-base-cased"


# *** To Fix ***


# *** tiny model issues ***
# missing model files:
UpperCamelCase_					=				"hf-internal-testing/tiny-random-mt5"
UpperCamelCase_					=				"hf-internal-testing/tiny-random-camembert"
UpperCamelCase_					=				"hf-internal-testing/tiny-random-openai-gpt"

# missing tokenizer files
UpperCamelCase_					=				"hf-internal-testing/tiny-random-convbert"
UpperCamelCase_					=				"hf-internal-testing/tiny-random-layoutlmv2"
UpperCamelCase_					=				"hf-internal-testing/tiny-random-hubert"

# issues with tokenizer
UpperCamelCase_					=				"hf-internal-testing/tiny-random-ctrl"
UpperCamelCase_					=				"hf-internal-testing/tiny-random-transfo-xl"  # same as ctrl

# other issues with tiny models
UpperCamelCase_					=				"hf-internal-testing/tiny-random-ibert"  # multiple issues with either mlm/qa/clas
UpperCamelCase_					=				"hf-internal-testing/tiny-random-reformer"  # multiple issues with either mlm/qa/clas

# *** Lacking official examples to test with ***
# or not working with examples
UpperCamelCase_					=				"hf-internal-testing/tiny-random-dpr"
# - "dpr"  examples/research_projects/rag-end2end-retriever/
UpperCamelCase_					=				"hf-internal-testing/tiny-random-rag"
# - "rag" research_projects
UpperCamelCase_					=				""
# - "luke" Entities classes - no plan to make such example
UpperCamelCase_					=				"hf-internal-testing/tiny-random-lxmert"
# - "lxmert" doesn't work with run_qa.py
UpperCamelCase_					=				"hf-internal-testing/tiny-random-clip"
# - "clip" nothing under pytorch examples - XXX: Suraj is working on adding some - check by end of Sep
UpperCamelCase_					=				"hf-internal-testing/tiny-random-speech_to_text"
# - "speech_to_text", nothing under pytorch examples


# *** Reactive mode ***
# models with low usage, unstable API, things about to change - do nothing about the following until someone runs into a problem
UpperCamelCase_					=				"hf-internal-testing/tiny-random-tapas"






def  lowerCAmelCase__		(					a_  :     Any=False			)   -> Tuple:
       UpperCAmelCase__	:   Optional[Any]												=	min(2			,						get_gpu_count()			) if distributed else 1
       UpperCAmelCase__	:   Optional[Any]												=	os.environ.get('''DS_TEST_PORT'''			,						_a			)
       return f"""deepspeed --num_nodes 1 --num_gpus {num_gpus} --master_port {master_port}""".split()






def  lowerCAmelCase__		(					)   -> List[str]:
       UpperCAmelCase__	:   Dict												=	f"""{FIXTURE_DIRECTORY}/tests_samples"""
       UpperCAmelCase__	:   Optional[int]												=	f"""{data_dir_samples}/wmt_en_ro"""
       UpperCAmelCase__	:   Dict												=	f"""{data_dir_samples}/xsum"""
       UpperCAmelCase__	:   Optional[Any]												=	"""
        --do_train
        --max_train_samples 4
        --per_device_train_batch_size 2
        --num_train_epochs 1
        --fp16
        --report_to none
        --overwrite_output_dir
        """.split()

       # try to cover as many models as possible once (it's enough to run on one task per model)
       # but need a tiny model for each
       #
       # should have "{model_type.upper()}_TINY" corresponding vars defined, e.g., T5_TINY, etc.
       UpperCAmelCase__	:   List[Any]												=	{
           """trans""": [
               """bart""",
               """fsmt""",
               """m2m_100""",
               """marian""",
               """mbart""",
               """t5""",
               """t5_v1""",
               # "mt5", missing model files
           ],
           """sum""": [
               """pegasus""",
           ],
           """clm""": [
               """big_bird""",
               """bigbird_pegasus""",
               """blenderbot""",
               """bloom""",
               """gpt2""",
               """gpt_neo""",
               """gptj""",
               """xlm-roberta""",
               """prophetnet""",
               # "camembert", missing model files
           ],
           """mlm""": [
               """albert""",
               """deberta""",
               """deberta-v2""",
               """distilbert""",
               """electra""",
               """flaubert""",
               """funnel""",
               """layoutlm""",
               # "reformer", # multiple issues with either mlm/qa/clas
           ],
           """qa""": [
               """led""",
               """longformer""",
               """mobilebert""",
               """mpnet""",
               """roberta""",
               """squeezebert""",
               # "convbert", # missing tokenizer files
               # "layoutlmv2", missing model files
           ],
           """clas""": [
               """bert""",
               """xlnet""",
               # "hubert", # missing tokenizer files
               # "ibert", # multiple issues with either mlm/qa/clas
               # "transfo-xl", # tokenizer issues as ctrl
               # "ctrl", # tokenizer issues
               # "openai-gpt", missing model files
               # "tapas", multiple issues
           ],
           """img_clas""": [
               """vit""",
           ],
       }

       UpperCAmelCase__	:   Union[str, Any]												=	f"""{ROOT_DIRECTORY}/examples/pytorch"""

       UpperCAmelCase__	:   List[str]												=	{
           """trans""": f"""
        {scripts_dir}/translation/run_translation.py
        --train_file {data_dir_wmt}/train.json
        --source_lang en
        --target_lang ro
        """,
           """sum""": f"""
        {scripts_dir}/summarization/run_summarization.py
        --train_file {data_dir_xsum}/sample.json
        --max_source_length 12
        --max_target_length 12
        --lang en
        """,
           """clm""": f"""
        {scripts_dir}/language-modeling/run_clm.py
        --train_file {FIXTURE_DIRECTORY}/sample_text.txt
        --block_size 8
        """,
           """mlm""": f"""
        {scripts_dir}/language-modeling/run_mlm.py
        --train_file {FIXTURE_DIRECTORY}/sample_text.txt
        """,
           """qa""": f"""
        {scripts_dir}/question-answering/run_qa.py
        --train_file {data_dir_samples}/SQUAD/sample.json
        """,
           """clas""": f"""
        {scripts_dir}/text-classification/run_glue.py
        --train_file {data_dir_samples}/MRPC/train.csv
        --max_seq_length 12
        --task_name MRPC
        """,
           """img_clas""": f"""
        {scripts_dir}/image-classification/run_image_classification.py
            --dataset_name hf-internal-testing/cats_vs_dogs_sample
            --remove_unused_columns False
            --max_steps 10
            --image_processor_name {DS_TESTS_DIRECTORY}/vit_feature_extractor.json
        """,
       }

       UpperCAmelCase__	:   List[Any]												=	get_launcher(distributed=_a			)

       UpperCAmelCase__	:   Tuple												=	{}
       for task, args in tasks.items():
              UpperCAmelCase__	:   Optional[int]												=	args.split()
              for model in tasksamodels[task]:
                     UpperCAmelCase__	:   Optional[Any]												=	globals()[f"""{model.upper().replace("-"			,						"_"			)}_TINY"""]
                     UpperCAmelCase__	:   str												=	f"""--model_name_or_path {model_name}""".split()
                     UpperCAmelCase__	:   Optional[Any]												=	launcher + args + args_model + args_main

                     # # generation special case
                     # if task == "gen":
                     #     launcher = f"deepspeed --num_nodes 1 --num_gpus 1".split()
                     #     args_model += f"--model_type {model}".split()
                     #     cmds[f"{task}_{model}"] = launcher + args + args_model
                     # else:

       return cmds


UpperCamelCase_					=				make_task_cmds()

UpperCamelCase_					=				"zero2"
UpperCamelCase_					=				"zero3"

UpperCamelCase_					=				[ZEROa, ZEROa]






def  lowerCAmelCase__		(					a_  :     List[str]			,						a_  :     int			,						a_  :     List[Any]			)   -> List[str]:
       UpperCAmelCase__	:   Optional[Any]												=	parameterized.to_safe_name('''_'''.join(str(_a			) for x in param.args			)			)
       return f"""{func.__name__}_{param_based_name}"""


# Cartesian-product of zero stages with models to test
UpperCamelCase_					=				list(itertools.product(stages, task_cmds.keys()))




@slow
@require_deepspeed
@require_torch_gpu
class 					__UpperCAmelCase (		__snake_case	):

      '''simple docstring'''





      def 							lowerCamelCase     (					self  ,		_UpperCAmelCase  ,		_UpperCAmelCase						):
             # return a ready to run train cmd
             if task not in task_cmds:
                    raise ValueError(F"""don\'t know of task {task}, have {task_cmds.keys()}"""						)

             UpperCAmelCase__	:   List[Any]												=	task_cmds[task]
             UpperCAmelCase__	:   Union[str, Any]												=	F"""--deepspeed {self.test_file_dir_str}/ds_config_{stage}.json""".split()

             UpperCAmelCase__	:   int												=	self.get_auto_remove_tmp_dir()
             UpperCAmelCase__	:   str												=	F"""--output_dir {output_dir}""".split()

             cmd += args_ds + args_out

             return cmd, output_dir




      @parameterized.expand(lowerCamelCase_  ,		name_func=lowerCamelCase_						)
      def 							lowerCamelCase     (					self  ,		_UpperCAmelCase  ,		_UpperCAmelCase						):
             # testing the ability to do a run followed by recovery of full fp32 weights

             UpperCAmelCase__	:   int												=	self.get_task_cmd(lowerCamelCase_  ,		lowerCamelCase_						)

             # 1. generate the checkpoint
             cmd += "--save_steps 1".split()
             # keep for quick debug
             # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] + cmd)); die
             execute_subprocess_async(lowerCamelCase_  ,		env=self.get_env()						)

             # 2. test that the fp32 weights get reconsolidated
             UpperCAmelCase__	:   str												=	F"""{output_dir}/checkpoint-1"""
             UpperCAmelCase__	:   Optional[int]												=	F"""{chkpt_dir}/out.bin"""
             UpperCAmelCase__	:   int												=	F"""{chkpt_dir}/zero_to_fp32.py {chkpt_dir} {recovered_model_path}"""
             # keep for quick debug
             # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
             subprocess.check_call(lowerCamelCase_  ,		shell=lowerCamelCase_						)
             assert os.path.exists(lowerCamelCase_						), F"""{recovered_model_path} was not found"""

             # possibly could also test that the resulting saved model is usable but given that we use
             # random models we won't know if it's any good




from ...configuration_utils import PretrainedConfig
from ...utils import is_detectrona_available, logging


UpperCamelCase_									=   logging.get_logger(__name__)

UpperCamelCase_									=   {
    '''layoutlmv2-base-uncased''': '''https://huggingface.co/microsoft/layoutlmv2-base-uncased/resolve/main/config.json''',
    '''layoutlmv2-large-uncased''': '''https://huggingface.co/microsoft/layoutlmv2-large-uncased/resolve/main/config.json''',
    # See all LayoutLMv2 models at https://huggingface.co/models?filter=layoutlmv2
}

# soft dependency
if is_detectrona_available():
      import detectrona







class       _snake_case				(     __snake_case ):






   '''simple docstring'''






   A__      : List[Any]       =						"layoutlmv2"


   def __init__(    self:     Tuple  ,lowerCamelCase_:     Dict=30522  ,lowerCamelCase_:     str=768  ,lowerCamelCase_:     Any=12  ,lowerCamelCase_:     Tuple=12  ,lowerCamelCase_:     int=3072  ,lowerCamelCase_:     int="gelu"  ,lowerCamelCase_:     Dict=0.1  ,lowerCamelCase_:     List[Any]=0.1  ,lowerCamelCase_:     List[Any]=512  ,lowerCamelCase_:     Tuple=2  ,lowerCamelCase_:     Dict=0.0_2  ,lowerCamelCase_:     Dict=1e-12  ,lowerCamelCase_:     Dict=0  ,lowerCamelCase_:     List[str]=1024  ,lowerCamelCase_:     Optional[Any]=128  ,lowerCamelCase_:     List[Any]=32  ,lowerCamelCase_:     List[Any]=True  ,lowerCamelCase_:     Dict=256  ,lowerCamelCase_:     int=64  ,lowerCamelCase_:     str=True  ,lowerCamelCase_:     Tuple=[7, 7, 256]  ,lowerCamelCase_:     Union[str, Any]=128  ,lowerCamelCase_:     Any=128  ,lowerCamelCase_:     int=True  ,lowerCamelCase_:     Union[str, Any]=True  ,lowerCamelCase_:     Optional[int]=False  ,lowerCamelCase_:     List[str]=None  ,**lowerCamelCase_:     str  ,)	->      Any:
       super().__init__(
           vocab_size=lowerCamelCase_  ,hidden_size=lowerCamelCase_  ,num_hidden_layers=lowerCamelCase_  ,num_attention_heads=lowerCamelCase_  ,intermediate_size=lowerCamelCase_  ,hidden_act=lowerCamelCase_  ,hidden_dropout_prob=lowerCamelCase_  ,attention_probs_dropout_prob=lowerCamelCase_  ,max_position_embeddings=lowerCamelCase_  ,type_vocab_size=lowerCamelCase_  ,initializer_range=lowerCamelCase_  ,layer_norm_eps=lowerCamelCase_  ,pad_token_id=lowerCamelCase_  ,**lowerCamelCase_  ,)
       UpperCAmelCase_     :       Optional[int]  		=					max_ad_position_embeddings
       UpperCAmelCase_     :       Optional[int]  		=					max_rel_pos
       UpperCAmelCase_     :       Optional[int]  		=					rel_pos_bins
       UpperCAmelCase_     :       List[str]  		=					fast_qkv
       UpperCAmelCase_     :       int  		=					max_rel_ad_pos
       UpperCAmelCase_     :       int  		=					rel_ad_pos_bins
       UpperCAmelCase_     :       Optional[Any]  		=					convert_sync_batchnorm
       UpperCAmelCase_     :       str  		=					image_feature_pool_shape
       UpperCAmelCase_     :       List[Any]  		=					coordinate_size
       UpperCAmelCase_     :       int  		=					shape_size
       UpperCAmelCase_     :       Optional[Any]  		=					has_relative_attention_bias
       UpperCAmelCase_     :       str  		=					has_spatial_attention_bias
       UpperCAmelCase_     :       Tuple  		=					has_visual_segment_embedding
       UpperCAmelCase_     :       Optional[Any]  		=					(
           detectrona_config_args if detectrona_config_args is not None else self.get_default_detectrona_config()
       )


   @classmethod
   def       A__    (    self:     Any							)	->      Dict:
       return {
           "MODEL.MASK_ON": True,
           "MODEL.PIXEL_STD": [5_7.3_7_5, 5_7.1_2_0, 5_8.3_9_5],
           "MODEL.BACKBONE.NAME": "build_resnet_fpn_backbone",
           "MODEL.FPN.IN_FEATURES": ["res2", "res3", "res4", "res5"],
           "MODEL.ANCHOR_GENERATOR.SIZES": [[32], [64], [128], [256], [512]],
           "MODEL.RPN.IN_FEATURES": ["p2", "p3", "p4", "p5", "p6"],
           "MODEL.RPN.PRE_NMS_TOPK_TRAIN": 2000,
           "MODEL.RPN.PRE_NMS_TOPK_TEST": 1000,
           "MODEL.RPN.POST_NMS_TOPK_TRAIN": 1000,
           "MODEL.POST_NMS_TOPK_TEST": 1000,
           "MODEL.ROI_HEADS.NAME": "StandardROIHeads",
           "MODEL.ROI_HEADS.NUM_CLASSES": 5,
           "MODEL.ROI_HEADS.IN_FEATURES": ["p2", "p3", "p4", "p5"],
           "MODEL.ROI_BOX_HEAD.NAME": "FastRCNNConvFCHead",
           "MODEL.ROI_BOX_HEAD.NUM_FC": 2,
           "MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION": 14,
           "MODEL.ROI_MASK_HEAD.NAME": "MaskRCNNConvUpsampleHead",
           "MODEL.ROI_MASK_HEAD.NUM_CONV": 4,
           "MODEL.ROI_MASK_HEAD.POOLER_RESOLUTION": 7,
           "MODEL.RESNETS.DEPTH": 101,
           "MODEL.RESNETS.SIZES": [[32], [64], [128], [256], [512]],
           "MODEL.RESNETS.ASPECT_RATIOS": [[0.5, 1.0, 2.0]],
           "MODEL.RESNETS.OUT_FEATURES": ["res2", "res3", "res4", "res5"],
           "MODEL.RESNETS.NUM_GROUPS": 32,
           "MODEL.RESNETS.WIDTH_PER_GROUP": 8,
           "MODEL.RESNETS.STRIDE_IN_1X1": False,
       }






   def       A__    (    self:     Tuple							)	->      int:
       UpperCAmelCase_     :       Any  		=					detectrona.config.get_cfg()
       for k, v in self.detectrona_config_args.items():
           UpperCAmelCase_     :       Optional[int]  		=					k.split("""."""							)
           UpperCAmelCase_     :       List[str]  		=					detectrona_config
           for attribute in attributes[:-1]:
               UpperCAmelCase_     :       Optional[Any]  		=					getattr(lowerCamelCase_  ,lowerCamelCase_							)
           setattr(lowerCamelCase_  ,attributes[-1]  ,lowerCamelCase_							)

       return detectrona_config
