
import argparse
import json
import logging
import os
import shutil
import sys
import tempfile
import unittest
from unittest import mock

import torch
from accelerate.utils import write_basic_config

from transformers.testing_utils import TestCasePlus, get_gpu_count, run_command, slow, torch_device
from transformers.utils import is_apex_available


logging.basicConfig(level=logging.DEBUG)

lowercase  :					Tuple						    =							logging.getLogger()







def 						_snake_case(  ) ->    Tuple:
       lowercase							:     Any       	=      argparse.ArgumentParser()
       parser.add_argument("""-f"""				)
       lowercase							:     Optional[int]       	=      parser.parse_args()
       return args.f







def 						_snake_case(  SCREAMING_SNAKE_CASE__				) ->    List[Any]:
       lowercase							:     Optional[int]       	=      {}
       lowercase							:     List[Any]       	=      os.path.join(SCREAMING_SNAKE_CASE__					,	"""all_results.json"""				)
       if os.path.exists(SCREAMING_SNAKE_CASE__				):
              with open(SCREAMING_SNAKE_CASE__					,	"""r"""				) as f:
                     lowercase							:     Optional[int]       	=      json.load(SCREAMING_SNAKE_CASE__				)
       else:
              raise ValueError(f"can't find {path}"				)
       return results







def 						_snake_case(  ) ->    Dict:
       lowercase							:     Any       	=      torch.cuda.is_available() and torch_device == """cuda"""
       return is_using_cuda and is_apex_available()


lowercase  :					List[Any]						    =							logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)




class __snake_case		(      lowerCAmelCase     ):
    @classmethod
    def       _SCREAMING_SNAKE_CASE       (  cls      ):



           '''simple docstring'''







           lowercase							:     Union[str, Any]       	=      tempfile.mkdtemp()
           lowercase							:     List[Any]       	=      os.path.join(cls.tmpdir		,"""default_config.yml"""      )
           write_basic_config(save_location=cls.configPath      )
           lowercase							:     int       	=      ["""accelerate""", """launch""", """--config_file""", cls.configPath]
    @classmethod
    def       _SCREAMING_SNAKE_CASE       (  cls      ):



           '''simple docstring'''







           shutil.rmtree(cls.tmpdir      )
    @mock.patch.dict(os.environ		,{"""WANDB_MODE""": """offline"""}      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     List[Any]       	=      self.get_auto_remove_tmp_dir()
           lowercase							:     List[str]       	=      f"\n            {self.examples_dir}/pytorch/text-classification/run_glue_no_trainer.py\n            --model_name_or_path distilbert-base-uncased\n            --output_dir {tmp_dir}\n            --train_file ./tests/fixtures/tests_samples/MRPC/train.csv\n            --validation_file ./tests/fixtures/tests_samples/MRPC/dev.csv\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --learning_rate=1e-4\n            --seed=42\n            --checkpointing_steps epoch\n            --with_tracking\n        ".split()

           if is_cuda_and_apex_available():
                  testargs.append("""--fp16"""      )

           run_command(self._launch_args + testargs      )
           lowercase							:     Dict       	=      get_results(snake_case      )
           self.assertGreaterEqual(result["""eval_accuracy"""]		,0.75      )
           self.assertTrue(os.path.exists(os.path.join(snake_case		,"""epoch_0"""      )      )      )
           self.assertTrue(os.path.exists(os.path.join(snake_case		,"""glue_no_trainer"""      )      )      )
    @mock.patch.dict(os.environ		,{"""WANDB_MODE""": """offline"""}      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     str       	=      self.get_auto_remove_tmp_dir()
           lowercase							:     Optional[Any]       	=      f"\n            {self.examples_dir}/pytorch/language-modeling/run_clm_no_trainer.py\n            --model_name_or_path distilgpt2\n            --train_file ./tests/fixtures/sample_text.txt\n            --validation_file ./tests/fixtures/sample_text.txt\n            --block_size 128\n            --per_device_train_batch_size 5\n            --per_device_eval_batch_size 5\n            --num_train_epochs 2\n            --output_dir {tmp_dir}\n            --checkpointing_steps epoch\n            --with_tracking\n        ".split()

           if torch.cuda.device_count() > 1:
                  # Skipping because there are not enough batches to train the model + would need a drop_last to work.
                  return

           run_command(self._launch_args + testargs      )
           lowercase							:     Tuple       	=      get_results(snake_case      )
           self.assertLess(result["""perplexity"""]		,100      )
           self.assertTrue(os.path.exists(os.path.join(snake_case		,"""epoch_0"""      )      )      )
           self.assertTrue(os.path.exists(os.path.join(snake_case		,"""clm_no_trainer"""      )      )      )
    @mock.patch.dict(os.environ		,{"""WANDB_MODE""": """offline"""}      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     List[str]       	=      self.get_auto_remove_tmp_dir()
           lowercase							:     Tuple       	=      f"\n            {self.examples_dir}/pytorch/language-modeling/run_mlm_no_trainer.py\n            --model_name_or_path distilroberta-base\n            --train_file ./tests/fixtures/sample_text.txt\n            --validation_file ./tests/fixtures/sample_text.txt\n            --output_dir {tmp_dir}\n            --num_train_epochs=1\n            --checkpointing_steps epoch\n            --with_tracking\n        ".split()

           run_command(self._launch_args + testargs      )
           lowercase							:     Dict       	=      get_results(snake_case      )
           self.assertLess(result["""perplexity"""]		,42      )
           self.assertTrue(os.path.exists(os.path.join(snake_case		,"""epoch_0"""      )      )      )
           self.assertTrue(os.path.exists(os.path.join(snake_case		,"""mlm_no_trainer"""      )      )      )
    @mock.patch.dict(os.environ		,{"""WANDB_MODE""": """offline"""}      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Optional[Any]       	=      7 if get_gpu_count() > 1 else 2

           lowercase							:     Dict       	=      self.get_auto_remove_tmp_dir()
           lowercase							:     Union[str, Any]       	=      f"\n            {self.examples_dir}/pytorch/token-classification/run_ner_no_trainer.py\n            --model_name_or_path bert-base-uncased\n            --train_file tests/fixtures/tests_samples/conll/sample.json\n            --validation_file tests/fixtures/tests_samples/conll/sample.json\n            --output_dir {tmp_dir}\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=2\n            --num_train_epochs={epochs}\n            --seed 7\n            --checkpointing_steps epoch\n            --with_tracking\n        ".split()

           run_command(self._launch_args + testargs      )
           lowercase							:     Optional[int]       	=      get_results(snake_case      )
           self.assertGreaterEqual(result["""eval_accuracy"""]		,0.75      )
           self.assertLess(result["""train_loss"""]		,0.5      )
           self.assertTrue(os.path.exists(os.path.join(snake_case		,"""epoch_0"""      )      )      )
           self.assertTrue(os.path.exists(os.path.join(snake_case		,"""ner_no_trainer"""      )      )      )
    @unittest.skip(reason="""Fix me @muellerzr"""      )
    @mock.patch.dict(os.environ		,{"""WANDB_MODE""": """offline"""}      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Tuple       	=      self.get_auto_remove_tmp_dir()
           lowercase							:     Tuple       	=      f"\n            {self.examples_dir}/pytorch/question-answering/run_qa_no_trainer.py\n            --model_name_or_path bert-base-uncased\n            --version_2_with_negative\n            --train_file tests/fixtures/tests_samples/SQUAD/sample.json\n            --validation_file tests/fixtures/tests_samples/SQUAD/sample.json\n            --output_dir {tmp_dir}\n            --seed=42\n            --max_train_steps=10\n            --num_warmup_steps=2\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --checkpointing_steps epoch\n            --with_tracking\n        ".split()

           run_command(self._launch_args + testargs      )
           lowercase							:     Union[str, Any]       	=      get_results(snake_case      )
           # Because we use --version_2_with_negative the testing script uses SQuAD v2 metrics.
           self.assertGreaterEqual(result["""eval_f1"""]		,28      )
           self.assertGreaterEqual(result["""eval_exact"""]		,28      )
           self.assertTrue(os.path.exists(os.path.join(snake_case		,"""epoch_0"""      )      )      )
           self.assertTrue(os.path.exists(os.path.join(snake_case		,"""qa_no_trainer"""      )      )      )
    @mock.patch.dict(os.environ		,{"""WANDB_MODE""": """offline"""}      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     int       	=      self.get_auto_remove_tmp_dir()
           lowercase							:     Optional[int]       	=      f"\n            {self.examples_dir}/pytorch/multiple-choice/run_swag_no_trainer.py\n            --model_name_or_path bert-base-uncased\n            --train_file tests/fixtures/tests_samples/swag/sample.json\n            --validation_file tests/fixtures/tests_samples/swag/sample.json\n            --output_dir {tmp_dir}\n            --max_train_steps=20\n            --num_warmup_steps=2\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --with_tracking\n        ".split()

           run_command(self._launch_args + testargs      )
           lowercase							:     Dict       	=      get_results(snake_case      )
           self.assertGreaterEqual(result["""eval_accuracy"""]		,0.8      )
           self.assertTrue(os.path.exists(os.path.join(snake_case		,"""swag_no_trainer"""      )      )      )
    @slow
    @mock.patch.dict(os.environ		,{"""WANDB_MODE""": """offline"""}      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Optional[int]       	=      self.get_auto_remove_tmp_dir()
           lowercase							:     Dict       	=      f"\n            {self.examples_dir}/pytorch/summarization/run_summarization_no_trainer.py\n            --model_name_or_path t5-small\n            --train_file tests/fixtures/tests_samples/xsum/sample.json\n            --validation_file tests/fixtures/tests_samples/xsum/sample.json\n            --output_dir {tmp_dir}\n            --max_train_steps=50\n            --num_warmup_steps=8\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --checkpointing_steps epoch\n            --with_tracking\n        ".split()

           run_command(self._launch_args + testargs      )
           lowercase							:     Tuple       	=      get_results(snake_case      )
           self.assertGreaterEqual(result["""eval_rouge1"""]		,10      )
           self.assertGreaterEqual(result["""eval_rouge2"""]		,2      )
           self.assertGreaterEqual(result["""eval_rougeL"""]		,7      )
           self.assertGreaterEqual(result["""eval_rougeLsum"""]		,7      )
           self.assertTrue(os.path.exists(os.path.join(snake_case		,"""epoch_0"""      )      )      )
           self.assertTrue(os.path.exists(os.path.join(snake_case		,"""summarization_no_trainer"""      )      )      )
    @slow
    @mock.patch.dict(os.environ		,{"""WANDB_MODE""": """offline"""}      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     List[str]       	=      self.get_auto_remove_tmp_dir()
           lowercase							:     Any       	=      f"\n            {self.examples_dir}/pytorch/translation/run_translation_no_trainer.py\n            --model_name_or_path sshleifer/student_marian_en_ro_6_1\n            --source_lang en\n            --target_lang ro\n            --train_file tests/fixtures/tests_samples/wmt16/sample.json\n            --validation_file tests/fixtures/tests_samples/wmt16/sample.json\n            --output_dir {tmp_dir}\n            --max_train_steps=50\n            --num_warmup_steps=8\n            --num_beams=6\n            --learning_rate=3e-3\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --source_lang en_XX\n            --target_lang ro_RO\n            --checkpointing_steps epoch\n            --with_tracking\n        ".split()

           run_command(self._launch_args + testargs      )
           lowercase							:     Optional[int]       	=      get_results(snake_case      )
           self.assertGreaterEqual(result["""eval_bleu"""]		,30      )
           self.assertTrue(os.path.exists(os.path.join(snake_case		,"""epoch_0"""      )      )      )
           self.assertTrue(os.path.exists(os.path.join(snake_case		,"""translation_no_trainer"""      )      )      )
    @slow
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Any       	=      logging.StreamHandler(sys.stdout      )
           logger.addHandler(snake_case      )

           lowercase							:     str       	=      self.get_auto_remove_tmp_dir()
           lowercase							:     Optional[int]       	=      f"\n            {self.examples_dir}/pytorch/semantic-segmentation/run_semantic_segmentation_no_trainer.py\n            --dataset_name huggingface/semantic-segmentation-test-sample\n            --output_dir {tmp_dir}\n            --max_train_steps=10\n            --num_warmup_steps=2\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --checkpointing_steps epoch\n        ".split()

           run_command(self._launch_args + testargs      )
           lowercase							:     Optional[int]       	=      get_results(snake_case      )
           self.assertGreaterEqual(result["""eval_overall_accuracy"""]		,0.10      )
    @mock.patch.dict(os.environ		,{"""WANDB_MODE""": """offline"""}      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Any       	=      self.get_auto_remove_tmp_dir()
           lowercase							:     Dict       	=      f"\n            {self.examples_dir}/pytorch/image-classification/run_image_classification_no_trainer.py\n            --model_name_or_path google/vit-base-patch16-224-in21k\n            --dataset_name hf-internal-testing/cats_vs_dogs_sample\n            --learning_rate 1e-4\n            --per_device_train_batch_size 2\n            --per_device_eval_batch_size 1\n            --max_train_steps 2\n            --train_val_split 0.1\n            --seed 42\n            --output_dir {tmp_dir}\n            --with_tracking\n            --checkpointing_steps 1\n        ".split()

           if is_cuda_and_apex_available():
                  testargs.append("""--fp16"""      )

           run_command(self._launch_args + testargs      )
           lowercase							:     Union[str, Any]       	=      get_results(snake_case      )
           # The base model scores a 25%
           self.assertGreaterEqual(result["""eval_accuracy"""]		,0.6      )
           self.assertTrue(os.path.exists(os.path.join(snake_case		,"""step_1"""      )      )      )
           self.assertTrue(os.path.exists(os.path.join(snake_case		,"""image_classification_no_trainer"""      )      )      )


import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
  import torch

if is_vision_available():
  from PIL import Image

  from transformers import MobileNetVaImageProcessor

class __snake_case		(      unittest.TestCase     ):
    def __init__(  self		,snake_case		,snake_case=7		,snake_case=3		,snake_case=18		,snake_case=30		,snake_case=400		,snake_case=True		,snake_case=None		,snake_case=True		,snake_case=None		,):



           '''simple docstring'''







           lowercase							:     Dict       	=      size if size is not None else {"""shortest_edge""": 20}
           lowercase							:     Union[str, Any]       	=      crop_size if crop_size is not None else {"""height""": 18, """width""": 18}
           lowercase							:     str       	=      parent
           lowercase							:     int       	=      batch_size
           lowercase							:     str       	=      num_channels
           lowercase							:     int       	=      image_size
           lowercase							:     List[str]       	=      min_resolution
           lowercase							:     str       	=      max_resolution
           lowercase							:     Dict       	=      do_resize
           lowercase							:     Dict       	=      size
           lowercase							:     Dict       	=      do_center_crop
           lowercase							:     str       	=      crop_size
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           return {
               "do_resize": self.do_resize,
               "size": self.size,
               "do_center_crop": self.do_center_crop,
               "crop_size": self.crop_size,
           }






@require_torch
@require_vision
class __snake_case		(      lowerCAmelCase					,	unittest.TestCase     ):
    _a		:						Any=				MobileNetVaImageProcessor if is_vision_available() else None
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     List[Any]       	=      MobileNetVaImageProcessingTester(self      )
    @property
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           return self.image_processor_tester.prepare_image_processor_dict()
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Optional[int]       	=      self.image_processing_class(**self.image_processor_dict      )
           self.assertTrue(hasattr(snake_case		,"""do_resize"""      )      )
           self.assertTrue(hasattr(snake_case		,"""size"""      )      )
           self.assertTrue(hasattr(snake_case		,"""do_center_crop"""      )      )
           self.assertTrue(hasattr(snake_case		,"""crop_size"""      )      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     int       	=      self.image_processing_class.from_dict(self.image_processor_dict      )
           self.assertEqual(image_processor.size		,{"""shortest_edge""": 20}      )
           self.assertEqual(image_processor.crop_size		,{"""height""": 18, """width""": 18}      )

           lowercase							:     int       	=      self.image_processing_class.from_dict(self.image_processor_dict		,size=42		,crop_size=84      )
           self.assertEqual(image_processor.size		,{"""shortest_edge""": 42}      )
           self.assertEqual(image_processor.crop_size		,{"""height""": 84, """width""": 84}      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           pass
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Optional[Any]       	=      self.image_processing_class(**self.image_processor_dict      )
           # create random PIL images
           lowercase							:     str       	=      prepare_image_inputs(self.image_processor_tester		,equal_resolution=snake_case      )
           for image in image_inputs:
                  self.assertIsInstance(snake_case		,Image.Image      )

           # Test not batched input
           lowercase							:     Dict       	=      image_processing(image_inputs[0]		,return_tensors="""pt"""      ).pixel_values
           self.assertEqual(
               encoded_images.shape		,(
                   1,
                   self.image_processor_tester.num_channels,
                   self.image_processor_tester.crop_size["""height"""],
                   self.image_processor_tester.crop_size["""width"""],
               )		,)

           # Test batched
           lowercase							:     Tuple       	=      image_processing(snake_case		,return_tensors="""pt"""      ).pixel_values
           self.assertEqual(
               encoded_images.shape		,(
                   self.image_processor_tester.batch_size,
                   self.image_processor_tester.num_channels,
                   self.image_processor_tester.crop_size["""height"""],
                   self.image_processor_tester.crop_size["""width"""],
               )		,)
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     List[Any]       	=      self.image_processing_class(**self.image_processor_dict      )
           # create random numpy tensors
           lowercase							:     Union[str, Any]       	=      prepare_image_inputs(self.image_processor_tester		,equal_resolution=snake_case		,numpify=snake_case      )
           for image in image_inputs:
                  self.assertIsInstance(snake_case		,np.ndarray      )

           # Test not batched input
           lowercase							:     Optional[Any]       	=      image_processing(image_inputs[0]		,return_tensors="""pt"""      ).pixel_values
           self.assertEqual(
               encoded_images.shape		,(
                   1,
                   self.image_processor_tester.num_channels,
                   self.image_processor_tester.crop_size["""height"""],
                   self.image_processor_tester.crop_size["""width"""],
               )		,)

           # Test batched
           lowercase							:     List[str]       	=      image_processing(snake_case		,return_tensors="""pt"""      ).pixel_values
           self.assertEqual(
               encoded_images.shape		,(
                   self.image_processor_tester.batch_size,
                   self.image_processor_tester.num_channels,
                   self.image_processor_tester.crop_size["""height"""],
                   self.image_processor_tester.crop_size["""width"""],
               )		,)
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Optional[Any]       	=      self.image_processing_class(**self.image_processor_dict      )
           # create random PyTorch tensors
           lowercase							:     Tuple       	=      prepare_image_inputs(self.image_processor_tester		,equal_resolution=snake_case		,torchify=snake_case      )
           for image in image_inputs:
                  self.assertIsInstance(snake_case		,torch.Tensor      )

           # Test not batched input
           lowercase							:     Optional[int]       	=      image_processing(image_inputs[0]		,return_tensors="""pt"""      ).pixel_values
           self.assertEqual(
               encoded_images.shape		,(
                   1,
                   self.image_processor_tester.num_channels,
                   self.image_processor_tester.crop_size["""height"""],
                   self.image_processor_tester.crop_size["""width"""],
               )		,)

           # Test batched
           lowercase							:     List[str]       	=      image_processing(snake_case		,return_tensors="""pt"""      ).pixel_values
           self.assertEqual(
               encoded_images.shape		,(
                   self.image_processor_tester.batch_size,
                   self.image_processor_tester.num_channels,
                   self.image_processor_tester.crop_size["""height"""],
                   self.image_processor_tester.crop_size["""width"""],
               )		,)
