




'''simple docstring'''





import itertools
import os
import subprocess
from os.path import dirname

from parameterized import parameterized

from tests.trainer.test_trainer import TrainerIntegrationCommon  # noqa
from transformers import is_torch_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    get_gpu_count,
    get_tests_dir,
    require_deepspeed,
    require_torch_gpu,
    slow,
)
from transformers.trainer_utils import set_seed


if is_torch_available():
       from tests.trainer.test_trainer import (  # noqa
           RegressionModelConfig,
           RegressionPreTrainedModel,
           get_regression_trainer,
       )


set_seed(4_2)

lowercase__								=						get_tests_dir('''fixtures''')
lowercase__								=						os.path.join(dirname(get_tests_dir()))
lowercase__								=						dirname(os.path.abspath(__file__))

# default torch.distributed port
lowercase__								=						'''10999'''

lowercase__								=						'''t5-small'''

# *** Working Models ***
lowercase__								=						'''hf-internal-testing/tiny-albert'''
lowercase__								=						'''sshleifer/bart-tiny-random'''
lowercase__								=						'''hf-internal-testing/tiny-bert'''
lowercase__								=						'''hf-internal-testing/tiny-random-bigbird_pegasus'''
lowercase__								=						'''hf-internal-testing/tiny-random-big_bird'''
lowercase__								=						'''hf-internal-testing/tiny-random-blenderbot'''
lowercase__								=						'''bigscience/bigscience-small-testing'''
lowercase__								=						'''hf-internal-testing/tiny-random-deberta'''
lowercase__								=						'''hf-internal-testing/tiny-random-deberta-v2'''
lowercase__								=						'''sshleifer/tiny-distilbert-base-cased'''
lowercase__								=						'''hf-internal-testing/tiny-electra'''
lowercase__								=						'''hf-internal-testing/tiny-random-flaubert'''
lowercase__								=						'''stas/tiny-wmt19-en-de'''
lowercase__								=						'''hf-internal-testing/tiny-random-funnel'''
lowercase__								=						'''sshleifer/tiny-gpt2'''
lowercase__								=						'''hf-internal-testing/tiny-random-gptj'''
lowercase__								=						'''hf-internal-testing/tiny-random-gpt_neo'''
lowercase__								=						'''hf-internal-testing/tiny-layoutlm'''
lowercase__								=						'''hf-internal-testing/tiny-random-led'''
lowercase__								=						'''hf-internal-testing/tiny-random-longformer'''
lowercase__								=						'''stas/tiny-m2m_100'''  # hf tiny model is unsuitable
lowercase__								=						'''sshleifer/tiny-marian-en-de'''
lowercase__								=						'''sshleifer/tiny-mbart'''
lowercase__								=						'''hf-internal-testing/tiny-random-mobilebert'''
lowercase__								=						'''hf-internal-testing/tiny-random-mpnet'''
lowercase__								=						'''stas/pegasus-cnn_dailymail-tiny-random'''
lowercase__								=						'''hf-internal-testing/tiny-random-prophetnet'''
lowercase__								=						'''sshleifer/tiny-distilroberta-base'''
lowercase__								=						'''hf-internal-testing/tiny-random-squeezebert'''
lowercase__								=						'''patrickvonplaten/t5-tiny-random'''
lowercase__								=						'''hf-internal-testing/tiny-random-t5-v1.1'''
lowercase__								=						'''hf-internal-testing/tiny-random-vit'''
lowercase__								=						'''hf-internal-testing/tiny-xlm-roberta'''
lowercase__								=						'''sshleifer/tiny-xlnet-base-cased'''


# *** To Fix ***


# *** tiny model issues ***
# missing model files:
lowercase__								=						'''hf-internal-testing/tiny-random-mt5'''
lowercase__								=						'''hf-internal-testing/tiny-random-camembert'''
lowercase__								=						'''hf-internal-testing/tiny-random-openai-gpt'''

# missing tokenizer files
lowercase__								=						'''hf-internal-testing/tiny-random-convbert'''
lowercase__								=						'''hf-internal-testing/tiny-random-layoutlmv2'''
lowercase__								=						'''hf-internal-testing/tiny-random-hubert'''

# issues with tokenizer
lowercase__								=						'''hf-internal-testing/tiny-random-ctrl'''
lowercase__								=						'''hf-internal-testing/tiny-random-transfo-xl'''  # same as ctrl

# other issues with tiny models
lowercase__								=						'''hf-internal-testing/tiny-random-ibert'''  # multiple issues with either mlm/qa/clas
lowercase__								=						'''hf-internal-testing/tiny-random-reformer'''  # multiple issues with either mlm/qa/clas

# *** Lacking official examples to test with ***
# or not working with examples
lowercase__								=						'''hf-internal-testing/tiny-random-dpr'''
# - "dpr"  examples/research_projects/rag-end2end-retriever/
lowercase__								=						'''hf-internal-testing/tiny-random-rag'''
# - "rag" research_projects
lowercase__								=						''''''
# - "luke" Entities classes - no plan to make such example
lowercase__								=						'''hf-internal-testing/tiny-random-lxmert'''
# - "lxmert" doesn't work with run_qa.py
lowercase__								=						'''hf-internal-testing/tiny-random-clip'''
# - "clip" nothing under pytorch examples - XXX: Suraj is working on adding some - check by end of Sep
lowercase__								=						'''hf-internal-testing/tiny-random-speech_to_text'''
# - "speech_to_text", nothing under pytorch examples


# *** Reactive mode ***
# models with low usage, unstable API, things about to change - do nothing about the following until someone runs into a problem
lowercase__								=						'''hf-internal-testing/tiny-random-tapas'''


def 					UpperCamelCase						(	snake_case=False ):




   '''simple docstring'''

   SCREAMING_SNAKE_CASE__      :					Tuple							    =			min(2						,  get_gpu_count() ) if distributed else 1
   SCREAMING_SNAKE_CASE__      :					Any							    =			os.environ.get('''DS_TEST_PORT'''						,  snake_case )
   return f"""deepspeed --num_nodes 1 --num_gpus {num_gpus} --master_port {master_port}""".split()


def 					UpperCamelCase						(	):




   '''simple docstring'''

   SCREAMING_SNAKE_CASE__      :					Dict							    =			f"""{FIXTURE_DIRECTORY}/tests_samples"""
   SCREAMING_SNAKE_CASE__      :					List[Any]							    =			f"""{data_dir_samples}/wmt_en_ro"""
   SCREAMING_SNAKE_CASE__      :					List[Any]							    =			f"""{data_dir_samples}/xsum"""
   SCREAMING_SNAKE_CASE__      :					Any							    =			'''
        --do_train
        --max_train_samples 4
        --per_device_train_batch_size 2
        --num_train_epochs 1
        --fp16
        --report_to none
        --overwrite_output_dir
        '''.split()

   # try to cover as many models as possible once (it's enough to run on one task per model)
   # but need a tiny model for each
   #
   # should have "{model_type.upper()}_TINY" corresponding vars defined, e.g., T5_TINY, etc.
   SCREAMING_SNAKE_CASE__      :					str							    =			{
       '''trans''': [
           '''bart''',
           '''fsmt''',
           '''m2m_100''',
           '''marian''',
           '''mbart''',
           '''t5''',
           '''t5_v1''',
           # "mt5", missing model files
       ],
       '''sum''': [
           '''pegasus''',
       ],
       '''clm''': [
           '''big_bird''',
           '''bigbird_pegasus''',
           '''blenderbot''',
           '''bloom''',
           '''gpt2''',
           '''gpt_neo''',
           '''gptj''',
           '''xlm-roberta''',
           '''prophetnet''',
           # "camembert", missing model files
       ],
       '''mlm''': [
           '''albert''',
           '''deberta''',
           '''deberta-v2''',
           '''distilbert''',
           '''electra''',
           '''flaubert''',
           '''funnel''',
           '''layoutlm''',
           # "reformer", # multiple issues with either mlm/qa/clas
       ],
       '''qa''': [
           '''led''',
           '''longformer''',
           '''mobilebert''',
           '''mpnet''',
           '''roberta''',
           '''squeezebert''',
           # "convbert", # missing tokenizer files
           # "layoutlmv2", missing model files
       ],
       '''clas''': [
           '''bert''',
           '''xlnet''',
           # "hubert", # missing tokenizer files
           # "ibert", # multiple issues with either mlm/qa/clas
           # "transfo-xl", # tokenizer issues as ctrl
           # "ctrl", # tokenizer issues
           # "openai-gpt", missing model files
           # "tapas", multiple issues
       ],
       '''img_clas''': [
           '''vit''',
       ],
   }

   SCREAMING_SNAKE_CASE__      :					List[Any]							    =			f"""{ROOT_DIRECTORY}/examples/pytorch"""

   SCREAMING_SNAKE_CASE__      :					Any							    =			{
       '''trans''': f"""
        {scripts_dir}/translation/run_translation.py
        --train_file {data_dir_wmt}/train.json
        --source_lang en
        --target_lang ro
        """,
       '''sum''': f"""
        {scripts_dir}/summarization/run_summarization.py
        --train_file {data_dir_xsum}/sample.json
        --max_source_length 12
        --max_target_length 12
        --lang en
        """,
       '''clm''': f"""
        {scripts_dir}/language-modeling/run_clm.py
        --train_file {FIXTURE_DIRECTORY}/sample_text.txt
        --block_size 8
        """,
       '''mlm''': f"""
        {scripts_dir}/language-modeling/run_mlm.py
        --train_file {FIXTURE_DIRECTORY}/sample_text.txt
        """,
       '''qa''': f"""
        {scripts_dir}/question-answering/run_qa.py
        --train_file {data_dir_samples}/SQUAD/sample.json
        """,
       '''clas''': f"""
        {scripts_dir}/text-classification/run_glue.py
        --train_file {data_dir_samples}/MRPC/train.csv
        --max_seq_length 12
        --task_name MRPC
        """,
       '''img_clas''': f"""
        {scripts_dir}/image-classification/run_image_classification.py
            --dataset_name hf-internal-testing/cats_vs_dogs_sample
            --remove_unused_columns False
            --max_steps 10
            --image_processor_name {DS_TESTS_DIRECTORY}/vit_feature_extractor.json
        """,
   }

   SCREAMING_SNAKE_CASE__      :					str							    =			get_launcher(distributed=snake_case )

   SCREAMING_SNAKE_CASE__      :					List[str]							    =			{}
   for task, args in tasks.items():
      SCREAMING_SNAKE_CASE__      :					Optional[Any]							    =			args.split()
      for model in tasksamodels[task]:
         SCREAMING_SNAKE_CASE__      :					Tuple							    =			globals()[f"""{model.upper().replace("-"						,  "_" )}_TINY"""]
         SCREAMING_SNAKE_CASE__      :					Optional[Any]							    =			f"""--model_name_or_path {model_name}""".split()
         SCREAMING_SNAKE_CASE__      :					List[str]							    =			launcher + args + args_model + args_main

         # # generation special case
         # if task == "gen":
         #     launcher = f"deepspeed --num_nodes 1 --num_gpus 1".split()
         #     args_model += f"--model_type {model}".split()
         #     cmds[f"{task}_{model}"] = launcher + args + args_model
         # else:

   return cmds


lowercase__								=						make_task_cmds()

lowercase__								=						'''zero2'''
lowercase__								=						'''zero3'''

lowercase__								=						[ZEROa, ZEROa]


def 					UpperCamelCase						(	snake_case						,  snake_case						,  snake_case ):




   '''simple docstring'''

   SCREAMING_SNAKE_CASE__      :					str							    =			parameterized.to_safe_name('''_'''.join(str(snake_case ) for x in param.args ) )
   return f"""{func.__name__}_{param_based_name}"""


# Cartesian-product of zero stages with models to test
lowercase__								=						list(itertools.product(stages, task_cmds.keys()))







@slow
@require_deepspeed
@require_torch_gpu
class     lowercase  ( UpperCAmelCase_):






      def  A__(self			,	a__			,	a__       )							-> Tuple:
         # return a ready to run train cmd
         if task not in task_cmds:
            raise ValueError(f"""don't know of task {task}, have {task_cmds.keys()}"""       )

         SCREAMING_SNAKE_CASE__      :					List[Any]							    =			task_cmds[task]
         SCREAMING_SNAKE_CASE__      :					Tuple							    =			f"""--deepspeed {self.test_file_dir_str}/ds_config_{stage}.json""".split()

         SCREAMING_SNAKE_CASE__      :					int							    =			self.get_auto_remove_tmp_dir()
         SCREAMING_SNAKE_CASE__      :					Optional[Any]							    =			f"""--output_dir {output_dir}""".split()

         cmd += args_ds + args_out

         return cmd, output_dir






      @parameterized.expand(a__			,	name_func=a__       )
      def  A__(self			,	a__			,	a__       )							-> Optional[int]:
         # testing the ability to do a run followed by recovery of full fp32 weights

         SCREAMING_SNAKE_CASE__    ,					SCREAMING_SNAKE_CASE__      :					Dict							    =			self.get_task_cmd(a__			,	a__       )

         # 1. generate the checkpoint
         cmd += "--save_steps 1".split()
         # keep for quick debug
         # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] + cmd)); die
         execute_subprocess_async(a__			,	env=self.get_env()       )

         # 2. test that the fp32 weights get reconsolidated
         SCREAMING_SNAKE_CASE__      :					Dict							    =			f"""{output_dir}/checkpoint-1"""
         SCREAMING_SNAKE_CASE__      :					str							    =			f"""{chkpt_dir}/out.bin"""
         SCREAMING_SNAKE_CASE__      :					Optional[Any]							    =			f"""{chkpt_dir}/zero_to_fp32.py {chkpt_dir} {recovered_model_path}"""
         # keep for quick debug
         # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
         subprocess.check_call(a__			,	shell=a__       )
         assert os.path.exists(a__       ), f"""{recovered_model_path} was not found"""

         # possibly could also test that the resulting saved model is usable but given that we use
         # random models we won't know if it's any good







'''simple docstring'''





import unittest

from transformers import (
    MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
    PreTrainedTokenizer,
    is_vision_available,
)
from transformers.pipelines import ImageClassificationPipeline, pipeline
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_or_tf,
    require_vision,
    slow,
)

from .test_pipelines_common import ANY


if is_vision_available():
       from PIL import Image
else:






       class     lowercase  :






             @staticmethod
             def  A__(*a__			,	**a__       )							-> Union[str, Any]:
                pass






@is_pipeline_test
@require_torch_or_tf
@require_vision
class     lowercase  ( unittest.TestCase):
      __lowercase					=	MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING
      __lowercase					=	TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING






      def  A__(self			,	a__			,	a__			,	a__       )							-> Dict:
         SCREAMING_SNAKE_CASE__      :					Optional[int]							    =			ImageClassificationPipeline(model=a__			,	image_processor=a__			,	top_k=2       )
         SCREAMING_SNAKE_CASE__      :					List[Any]							    =			[
             Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''       ),
             '''http://images.cocodataset.org/val2017/000000039769.jpg''',
         ]
         return image_classifier, examples






      def  A__(self			,	a__			,	a__       )							-> List[Any]:
         SCREAMING_SNAKE_CASE__      :					int							    =			image_classifier('''./tests/fixtures/tests_samples/COCO/000000039769.png'''       )

         self.assertEqual(
             a__			,	[
                 {'''score''': ANY(a__       ), '''label''': ANY(a__       )},
                 {'''score''': ANY(a__       ), '''label''': ANY(a__       )},
             ]			,	)

         import datasets

         SCREAMING_SNAKE_CASE__      :					Optional[Any]							    =			datasets.load_dataset('''hf-internal-testing/fixtures_image_utils'''			,	'''image'''			,	split='''test'''       )

         # Accepts URL + PIL.Image + lists
         SCREAMING_SNAKE_CASE__      :					Tuple							    =			image_classifier(
             [
                 Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''       ),
                 '''http://images.cocodataset.org/val2017/000000039769.jpg''',
                 # RGBA
                 dataset[0]['''file'''],
                 # LA
                 dataset[1]['''file'''],
                 # L
                 dataset[2]['''file'''],
             ]       )
         self.assertEqual(
             a__			,	[
                 [
                     {'''score''': ANY(a__       ), '''label''': ANY(a__       )},
                     {'''score''': ANY(a__       ), '''label''': ANY(a__       )},
                 ],
                 [
                     {'''score''': ANY(a__       ), '''label''': ANY(a__       )},
                     {'''score''': ANY(a__       ), '''label''': ANY(a__       )},
                 ],
                 [
                     {'''score''': ANY(a__       ), '''label''': ANY(a__       )},
                     {'''score''': ANY(a__       ), '''label''': ANY(a__       )},
                 ],
                 [
                     {'''score''': ANY(a__       ), '''label''': ANY(a__       )},
                     {'''score''': ANY(a__       ), '''label''': ANY(a__       )},
                 ],
                 [
                     {'''score''': ANY(a__       ), '''label''': ANY(a__       )},
                     {'''score''': ANY(a__       ), '''label''': ANY(a__       )},
                 ],
             ]			,	)






      @require_torch
      def  A__(self       )							-> List[str]:
         SCREAMING_SNAKE_CASE__      :					List[str]							    =			'''hf-internal-testing/tiny-random-vit'''
         SCREAMING_SNAKE_CASE__      :					Any							    =			pipeline('''image-classification'''			,	model=a__       )

         SCREAMING_SNAKE_CASE__      :					Any							    =			image_classifier('''http://images.cocodataset.org/val2017/000000039769.jpg'''       )
         self.assertEqual(
             nested_simplify(a__			,	decimals=4       )			,	[{'''label''': '''LABEL_1''', '''score''': 0.5_74}, {'''label''': '''LABEL_0''', '''score''': 0.4_26}]			,	)

         SCREAMING_SNAKE_CASE__      :					Any							    =			image_classifier(
             [
                 '''http://images.cocodataset.org/val2017/000000039769.jpg''',
                 '''http://images.cocodataset.org/val2017/000000039769.jpg''',
             ]			,	top_k=2			,	)
         self.assertEqual(
             nested_simplify(a__			,	decimals=4       )			,	[
                 [{'''label''': '''LABEL_1''', '''score''': 0.5_74}, {'''label''': '''LABEL_0''', '''score''': 0.4_26}],
                 [{'''label''': '''LABEL_1''', '''score''': 0.5_74}, {'''label''': '''LABEL_0''', '''score''': 0.4_26}],
             ]			,	)






      @require_tf
      def  A__(self       )							-> Optional[int]:
         SCREAMING_SNAKE_CASE__      :					Optional[Any]							    =			'''hf-internal-testing/tiny-random-vit'''
         SCREAMING_SNAKE_CASE__      :					Optional[Any]							    =			pipeline('''image-classification'''			,	model=a__			,	framework='''tf'''       )

         SCREAMING_SNAKE_CASE__      :					List[str]							    =			image_classifier('''http://images.cocodataset.org/val2017/000000039769.jpg'''       )
         self.assertEqual(
             nested_simplify(a__			,	decimals=4       )			,	[{'''label''': '''LABEL_1''', '''score''': 0.5_74}, {'''label''': '''LABEL_0''', '''score''': 0.4_26}]			,	)

         SCREAMING_SNAKE_CASE__      :					Dict							    =			image_classifier(
             [
                 '''http://images.cocodataset.org/val2017/000000039769.jpg''',
                 '''http://images.cocodataset.org/val2017/000000039769.jpg''',
             ]			,	top_k=2			,	)
         self.assertEqual(
             nested_simplify(a__			,	decimals=4       )			,	[
                 [{'''label''': '''LABEL_1''', '''score''': 0.5_74}, {'''label''': '''LABEL_0''', '''score''': 0.4_26}],
                 [{'''label''': '''LABEL_1''', '''score''': 0.5_74}, {'''label''': '''LABEL_0''', '''score''': 0.4_26}],
             ]			,	)






      def  A__(self       )							-> Union[str, Any]:
         SCREAMING_SNAKE_CASE__      :					Optional[Any]							    =			PreTrainedTokenizer()

         # Assert that the pipeline can be initialized with a feature extractor that is not in any mapping
         SCREAMING_SNAKE_CASE__      :					Optional[int]							    =			pipeline(
             '''image-classification'''			,	model='''hf-internal-testing/tiny-random-vit'''			,	tokenizer=a__       )

         self.assertIs(image_classifier.tokenizer			,	a__       )






      @slow
      @require_torch
      def  A__(self       )							-> int:
         # Perceiver is not tested by `run_pipeline_test` properly.
         # That is because the type of feature_extractor and model preprocessor need to be kept
         # in sync, which is not the case in the current design
         SCREAMING_SNAKE_CASE__      :					Union[str, Any]							    =			pipeline('''image-classification'''			,	model='''deepmind/vision-perceiver-conv'''       )
         SCREAMING_SNAKE_CASE__      :					str							    =			image_classifier('''http://images.cocodataset.org/val2017/000000039769.jpg'''       )
         self.assertEqual(
             nested_simplify(a__			,	decimals=4       )			,	[
                 {'''score''': 0.43_85, '''label''': '''tabby, tabby cat'''},
                 {'''score''': 0.3_21, '''label''': '''tiger cat'''},
                 {'''score''': 0.05_02, '''label''': '''Egyptian cat'''},
                 {'''score''': 0.01_37, '''label''': '''crib, cot'''},
                 {'''score''': 0.0_07, '''label''': '''radiator'''},
             ]			,	)

         SCREAMING_SNAKE_CASE__      :					str							    =			pipeline('''image-classification'''			,	model='''deepmind/vision-perceiver-fourier'''       )
         SCREAMING_SNAKE_CASE__      :					Optional[Any]							    =			image_classifier('''http://images.cocodataset.org/val2017/000000039769.jpg'''       )
         self.assertEqual(
             nested_simplify(a__			,	decimals=4       )			,	[
                 {'''score''': 0.56_58, '''label''': '''tabby, tabby cat'''},
                 {'''score''': 0.13_09, '''label''': '''tiger cat'''},
                 {'''score''': 0.07_22, '''label''': '''Egyptian cat'''},
                 {'''score''': 0.07_07, '''label''': '''remote control, remote'''},
                 {'''score''': 0.00_82, '''label''': '''computer keyboard, keypad'''},
             ]			,	)

         SCREAMING_SNAKE_CASE__      :					Dict							    =			pipeline('''image-classification'''			,	model='''deepmind/vision-perceiver-learned'''       )
         SCREAMING_SNAKE_CASE__      :					Optional[int]							    =			image_classifier('''http://images.cocodataset.org/val2017/000000039769.jpg'''       )
         self.assertEqual(
             nested_simplify(a__			,	decimals=4       )			,	[
                 {'''score''': 0.30_22, '''label''': '''tabby, tabby cat'''},
                 {'''score''': 0.23_62, '''label''': '''Egyptian cat'''},
                 {'''score''': 0.18_56, '''label''': '''tiger cat'''},
                 {'''score''': 0.03_24, '''label''': '''remote control, remote'''},
                 {'''score''': 0.00_96, '''label''': '''quilt, comforter, comfort, puff'''},
             ]			,	)

