





"""simple docstring"""





import argparse
import json
import logging
import os
import shutil
import sys
import tempfile
import unittest
from unittest import mock

import torch
from accelerate.utils import write_basic_config

from transformers.testing_utils import TestCasePlus, get_gpu_count, run_command, slow, torch_device
from transformers.utils import is_apex_available


logging.basicConfig(level=logging.DEBUG)

SCREAMING_SNAKE_CASE__					=   logging.getLogger()


def 			UpperCAmelCase__							(  ):







      '''simple docstring'''




      lowerCAmelCase  						=       argparse.ArgumentParser()
      parser.add_argument("""-f"""			)
      lowerCAmelCase  						=       parser.parse_args()
      return args.f


def 			UpperCAmelCase__							(  SCREAMING_SNAKE_CASE     :						int			):







      '''simple docstring'''




      lowerCAmelCase  						=       {}
      lowerCAmelCase  						=       os.path.join(SCREAMING_SNAKE_CASE							,						"""all_results.json"""			)
      if os.path.exists(SCREAMING_SNAKE_CASE			):
            with open(SCREAMING_SNAKE_CASE							,						"""r"""			) as f:
                  lowerCAmelCase  						=       json.load(SCREAMING_SNAKE_CASE			)
      else:
            raise ValueError(F'can\'t find {path}'			)
      return results


def 			UpperCAmelCase__							(  ):







      '''simple docstring'''




      lowerCAmelCase  						=       torch.cuda.is_available() and torch_device == """cuda"""
      return is_using_cuda and is_apex_available()


SCREAMING_SNAKE_CASE__					=   logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)






class 		lowercase					(					_UpperCAmelCase					):





      @classmethod
      def     _snake_case			(       cls )		->     List[str]:
            # Write Accelerate config, will pick up on CPU, GPU, and multi-GPU
            lowerCAmelCase  						=       tempfile.mkdtemp()
            lowerCAmelCase  						=       os.path.join(cls.tmpdir      ,						"""default_config.yml""" )
            write_basic_config(save_location=cls.configPath )
            lowerCAmelCase  						=       ["""accelerate""", """launch""", """--config_file""", cls.configPath]





      @classmethod
      def     _snake_case			(       cls )		->     List[Any]:
            shutil.rmtree(cls.tmpdir )





      @mock.patch.dict(os.environ      ,						{"""WANDB_MODE""": """offline"""} )
      def     _snake_case			(       self )		->     Union[str, Any]:
            lowerCAmelCase  						=       self.get_auto_remove_tmp_dir()
            lowerCAmelCase  						=       f'\n            {self.examples_dir}/pytorch/text-classification/run_glue_no_trainer.py\n            --model_name_or_path distilbert-base-uncased\n            --output_dir {tmp_dir}\n            --train_file ./tests/fixtures/tests_samples/MRPC/train.csv\n            --validation_file ./tests/fixtures/tests_samples/MRPC/dev.csv\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --learning_rate=1e-4\n            --seed=42\n            --checkpointing_steps epoch\n            --with_tracking\n        '.split()

            if is_cuda_and_apex_available():
                  testargs.append("""--fp16""" )

            run_command(self._launch_args + testargs )
            lowerCAmelCase  						=       get_results(lowercase )
            self.assertGreaterEqual(result["""eval_accuracy"""]      ,						0.75 )
            self.assertTrue(os.path.exists(os.path.join(lowercase      ,						"""epoch_0""" ) ) )
            self.assertTrue(os.path.exists(os.path.join(lowercase      ,						"""glue_no_trainer""" ) ) )





      @mock.patch.dict(os.environ      ,						{"""WANDB_MODE""": """offline"""} )
      def     _snake_case			(       self )		->     int:
            lowerCAmelCase  						=       self.get_auto_remove_tmp_dir()
            lowerCAmelCase  						=       f'\n            {self.examples_dir}/pytorch/language-modeling/run_clm_no_trainer.py\n            --model_name_or_path distilgpt2\n            --train_file ./tests/fixtures/sample_text.txt\n            --validation_file ./tests/fixtures/sample_text.txt\n            --block_size 128\n            --per_device_train_batch_size 5\n            --per_device_eval_batch_size 5\n            --num_train_epochs 2\n            --output_dir {tmp_dir}\n            --checkpointing_steps epoch\n            --with_tracking\n        '.split()

            if torch.cuda.device_count() > 1:
                  # Skipping because there are not enough batches to train the model + would need a drop_last to work.
                  return

            run_command(self._launch_args + testargs )
            lowerCAmelCase  						=       get_results(lowercase )
            self.assertLess(result["""perplexity"""]      ,						100 )
            self.assertTrue(os.path.exists(os.path.join(lowercase      ,						"""epoch_0""" ) ) )
            self.assertTrue(os.path.exists(os.path.join(lowercase      ,						"""clm_no_trainer""" ) ) )





      @mock.patch.dict(os.environ      ,						{"""WANDB_MODE""": """offline"""} )
      def     _snake_case			(       self )		->     Tuple:
            lowerCAmelCase  						=       self.get_auto_remove_tmp_dir()
            lowerCAmelCase  						=       f'\n            {self.examples_dir}/pytorch/language-modeling/run_mlm_no_trainer.py\n            --model_name_or_path distilroberta-base\n            --train_file ./tests/fixtures/sample_text.txt\n            --validation_file ./tests/fixtures/sample_text.txt\n            --output_dir {tmp_dir}\n            --num_train_epochs=1\n            --checkpointing_steps epoch\n            --with_tracking\n        '.split()

            run_command(self._launch_args + testargs )
            lowerCAmelCase  						=       get_results(lowercase )
            self.assertLess(result["""perplexity"""]      ,						42 )
            self.assertTrue(os.path.exists(os.path.join(lowercase      ,						"""epoch_0""" ) ) )
            self.assertTrue(os.path.exists(os.path.join(lowercase      ,						"""mlm_no_trainer""" ) ) )





      @mock.patch.dict(os.environ      ,						{"""WANDB_MODE""": """offline"""} )
      def     _snake_case			(       self )		->     Dict:
            # with so little data distributed training needs more epochs to get the score on par with 0/1 gpu
            lowerCAmelCase  						=       7 if get_gpu_count() > 1 else 2

            lowerCAmelCase  						=       self.get_auto_remove_tmp_dir()
            lowerCAmelCase  						=       f'\n            {self.examples_dir}/pytorch/token-classification/run_ner_no_trainer.py\n            --model_name_or_path bert-base-uncased\n            --train_file tests/fixtures/tests_samples/conll/sample.json\n            --validation_file tests/fixtures/tests_samples/conll/sample.json\n            --output_dir {tmp_dir}\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=2\n            --num_train_epochs={epochs}\n            --seed 7\n            --checkpointing_steps epoch\n            --with_tracking\n        '.split()

            run_command(self._launch_args + testargs )
            lowerCAmelCase  						=       get_results(lowercase )
            self.assertGreaterEqual(result["""eval_accuracy"""]      ,						0.75 )
            self.assertLess(result["""train_loss"""]      ,						0.5 )
            self.assertTrue(os.path.exists(os.path.join(lowercase      ,						"""epoch_0""" ) ) )
            self.assertTrue(os.path.exists(os.path.join(lowercase      ,						"""ner_no_trainer""" ) ) )





      @unittest.skip(reason="""Fix me @muellerzr""" )
      @mock.patch.dict(os.environ      ,						{"""WANDB_MODE""": """offline"""} )
      def     _snake_case			(       self )		->     List[str]:
            lowerCAmelCase  						=       self.get_auto_remove_tmp_dir()
            lowerCAmelCase  						=       f'\n            {self.examples_dir}/pytorch/question-answering/run_qa_no_trainer.py\n            --model_name_or_path bert-base-uncased\n            --version_2_with_negative\n            --train_file tests/fixtures/tests_samples/SQUAD/sample.json\n            --validation_file tests/fixtures/tests_samples/SQUAD/sample.json\n            --output_dir {tmp_dir}\n            --seed=42\n            --max_train_steps=10\n            --num_warmup_steps=2\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --checkpointing_steps epoch\n            --with_tracking\n        '.split()

            run_command(self._launch_args + testargs )
            lowerCAmelCase  						=       get_results(lowercase )
            # Because we use --version_2_with_negative the testing script uses SQuAD v2 metrics.
            self.assertGreaterEqual(result["""eval_f1"""]      ,						28 )
            self.assertGreaterEqual(result["""eval_exact"""]      ,						28 )
            self.assertTrue(os.path.exists(os.path.join(lowercase      ,						"""epoch_0""" ) ) )
            self.assertTrue(os.path.exists(os.path.join(lowercase      ,						"""qa_no_trainer""" ) ) )





      @mock.patch.dict(os.environ      ,						{"""WANDB_MODE""": """offline"""} )
      def     _snake_case			(       self )		->     List[Any]:
            lowerCAmelCase  						=       self.get_auto_remove_tmp_dir()
            lowerCAmelCase  						=       f'\n            {self.examples_dir}/pytorch/multiple-choice/run_swag_no_trainer.py\n            --model_name_or_path bert-base-uncased\n            --train_file tests/fixtures/tests_samples/swag/sample.json\n            --validation_file tests/fixtures/tests_samples/swag/sample.json\n            --output_dir {tmp_dir}\n            --max_train_steps=20\n            --num_warmup_steps=2\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --with_tracking\n        '.split()

            run_command(self._launch_args + testargs )
            lowerCAmelCase  						=       get_results(lowercase )
            self.assertGreaterEqual(result["""eval_accuracy"""]      ,						0.8 )
            self.assertTrue(os.path.exists(os.path.join(lowercase      ,						"""swag_no_trainer""" ) ) )





      @slow
      @mock.patch.dict(os.environ      ,						{"""WANDB_MODE""": """offline"""} )
      def     _snake_case			(       self )		->     List[str]:
            lowerCAmelCase  						=       self.get_auto_remove_tmp_dir()
            lowerCAmelCase  						=       f'\n            {self.examples_dir}/pytorch/summarization/run_summarization_no_trainer.py\n            --model_name_or_path t5-small\n            --train_file tests/fixtures/tests_samples/xsum/sample.json\n            --validation_file tests/fixtures/tests_samples/xsum/sample.json\n            --output_dir {tmp_dir}\n            --max_train_steps=50\n            --num_warmup_steps=8\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --checkpointing_steps epoch\n            --with_tracking\n        '.split()

            run_command(self._launch_args + testargs )
            lowerCAmelCase  						=       get_results(lowercase )
            self.assertGreaterEqual(result["""eval_rouge1"""]      ,						10 )
            self.assertGreaterEqual(result["""eval_rouge2"""]      ,						2 )
            self.assertGreaterEqual(result["""eval_rougeL"""]      ,						7 )
            self.assertGreaterEqual(result["""eval_rougeLsum"""]      ,						7 )
            self.assertTrue(os.path.exists(os.path.join(lowercase      ,						"""epoch_0""" ) ) )
            self.assertTrue(os.path.exists(os.path.join(lowercase      ,						"""summarization_no_trainer""" ) ) )





      @slow
      @mock.patch.dict(os.environ      ,						{"""WANDB_MODE""": """offline"""} )
      def     _snake_case			(       self )		->     str:
            lowerCAmelCase  						=       self.get_auto_remove_tmp_dir()
            lowerCAmelCase  						=       f'\n            {self.examples_dir}/pytorch/translation/run_translation_no_trainer.py\n            --model_name_or_path sshleifer/student_marian_en_ro_6_1\n            --source_lang en\n            --target_lang ro\n            --train_file tests/fixtures/tests_samples/wmt16/sample.json\n            --validation_file tests/fixtures/tests_samples/wmt16/sample.json\n            --output_dir {tmp_dir}\n            --max_train_steps=50\n            --num_warmup_steps=8\n            --num_beams=6\n            --learning_rate=3e-3\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --source_lang en_XX\n            --target_lang ro_RO\n            --checkpointing_steps epoch\n            --with_tracking\n        '.split()

            run_command(self._launch_args + testargs )
            lowerCAmelCase  						=       get_results(lowercase )
            self.assertGreaterEqual(result["""eval_bleu"""]      ,						30 )
            self.assertTrue(os.path.exists(os.path.join(lowercase      ,						"""epoch_0""" ) ) )
            self.assertTrue(os.path.exists(os.path.join(lowercase      ,						"""translation_no_trainer""" ) ) )





      @slow
      def     _snake_case			(       self )		->     Optional[int]:
            lowerCAmelCase  						=       logging.StreamHandler(sys.stdout )
            logger.addHandler(lowercase )

            lowerCAmelCase  						=       self.get_auto_remove_tmp_dir()
            lowerCAmelCase  						=       f'\n            {self.examples_dir}/pytorch/semantic-segmentation/run_semantic_segmentation_no_trainer.py\n            --dataset_name huggingface/semantic-segmentation-test-sample\n            --output_dir {tmp_dir}\n            --max_train_steps=10\n            --num_warmup_steps=2\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --checkpointing_steps epoch\n        '.split()

            run_command(self._launch_args + testargs )
            lowerCAmelCase  						=       get_results(lowercase )
            self.assertGreaterEqual(result["""eval_overall_accuracy"""]      ,						0.10 )





      @mock.patch.dict(os.environ      ,						{"""WANDB_MODE""": """offline"""} )
      def     _snake_case			(       self )		->     Union[str, Any]:
            lowerCAmelCase  						=       self.get_auto_remove_tmp_dir()
            lowerCAmelCase  						=       f'\n            {self.examples_dir}/pytorch/image-classification/run_image_classification_no_trainer.py\n            --model_name_or_path google/vit-base-patch16-224-in21k\n            --dataset_name hf-internal-testing/cats_vs_dogs_sample\n            --learning_rate 1e-4\n            --per_device_train_batch_size 2\n            --per_device_eval_batch_size 1\n            --max_train_steps 2\n            --train_val_split 0.1\n            --seed 42\n            --output_dir {tmp_dir}\n            --with_tracking\n            --checkpointing_steps 1\n        '.split()

            if is_cuda_and_apex_available():
                  testargs.append("""--fp16""" )

            run_command(self._launch_args + testargs )
            lowerCAmelCase  						=       get_results(lowercase )
            # The base model scores a 25%
            self.assertGreaterEqual(result["""eval_accuracy"""]      ,						0.6 )
            self.assertTrue(os.path.exists(os.path.join(lowercase      ,						"""step_1""" ) ) )
            self.assertTrue(os.path.exists(os.path.join(lowercase      ,						"""image_classification_no_trainer""" ) ) )













"""simple docstring"""





import re
import string

import numpy as np

import datasets


SCREAMING_SNAKE_CASE__					=   "\nReturns the rate at which the input predicted strings exactly match their references, ignoring any strings input as part of the regexes_to_ignore list.\n"

SCREAMING_SNAKE_CASE__					=   "\nArgs:\n    predictions: List of predicted texts.\n    references: List of reference texts.\n    regexes_to_ignore: List, defaults to None. Regex expressions of characters to\n        ignore when calculating the exact matches. Note: these regexes are removed\n        from the input data before the changes based on the options below (e.g. ignore_case,\n        ignore_punctuation, ignore_numbers) are applied.\n    ignore_case: Boolean, defaults to False. If true, turns everything\n        to lowercase so that capitalization differences are ignored.\n    ignore_punctuation: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\n    ignore_numbers: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\nReturns:\n    exact_match: Dictionary containing exact_match rate. Possible values are between 0.0 and 100.0, inclusive.\nExamples:\n    >>> exact_match = datasets.load_metric(\"exact_match\")\n    >>> refs = [\"the cat\", \"theater\", \"YELLING\", \"agent007\"]\n    >>> preds = [\"cat?\", \"theater\", \"yelling\", \"agent\"]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results[\"exact_match\"], 1))\n    25.0\n\n    >>> exact_match = datasets.load_metric(\"exact_match\")\n    >>> refs = [\"the cat\", \"theater\", \"YELLING\", \"agent007\"]\n    >>> preds = [\"cat?\", \"theater\", \"yelling\", \"agent\"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=[\"the \", \"yell\"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results[\"exact_match\"], 1))\n    50.0\n\n\n    >>> exact_match = datasets.load_metric(\"exact_match\")\n    >>> refs = [\"the cat\", \"theater\", \"YELLING\", \"agent007\"]\n    >>> preds = [\"cat?\", \"theater\", \"yelling\", \"agent\"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=[\"the \", \"yell\", \"YELL\"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results[\"exact_match\"], 1))\n    75.0\n\n    >>> exact_match = datasets.load_metric(\"exact_match\")\n    >>> refs = [\"the cat\", \"theater\", \"YELLING\", \"agent007\"]\n    >>> preds = [\"cat?\", \"theater\", \"yelling\", \"agent\"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=[\"the \", \"yell\", \"YELL\"], ignore_case=True, ignore_punctuation=True, ignore_numbers=True)\n    >>> print(round(results[\"exact_match\"], 1))\n    100.0\n\n    >>> exact_match = datasets.load_metric(\"exact_match\")\n    >>> refs = [\"The cat sat on the mat.\", \"Theaters are great.\", \"It's like comparing oranges and apples.\"]\n    >>> preds = [\"The cat sat on the mat?\", \"Theaters are great.\", \"It's like comparing apples and oranges.\"]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results[\"exact_match\"], 1))\n    33.3\n\n"

SCREAMING_SNAKE_CASE__					=   "\n"

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION     ,  _KWARGS_DESCRIPTION					)
class 		lowercase					(					datasets.Metric					):





      def     _snake_case			(       self )		->     Optional[int]:
            return datasets.MetricInfo(
                description=_DESCRIPTION      ,						citation=_CITATION      ,						inputs_description=_KWARGS_DESCRIPTION      ,						features=datasets.Features(
                    {
                        """predictions""": datasets.Value("""string"""      ,						id="""sequence""" ),
                        """references""": datasets.Value("""string"""      ,						id="""sequence""" ),
                    } )      ,						reference_urls=[]      ,						)





      def     _snake_case			(       self      ,						lowercase      ,						lowercase      ,						lowercase=None      ,						lowercase=False      ,						lowercase=False      ,						lowercase=False      ,						)		->     Optional[Any]:
            if regexes_to_ignore is not None:
                  for s in regexes_to_ignore:
                        lowerCAmelCase  						=       np.array([re.sub(lowercase      ,						""""""      ,						lowercase ) for x in predictions] )
                        lowerCAmelCase  						=       np.array([re.sub(lowercase      ,						""""""      ,						lowercase ) for x in references] )
            else:
                  lowerCAmelCase  						=       np.asarray(lowercase )
                  lowerCAmelCase  						=       np.asarray(lowercase )

            if ignore_case:
                  lowerCAmelCase  						=       np.char.lower(lowercase )
                  lowerCAmelCase  						=       np.char.lower(lowercase )

            if ignore_punctuation:
                  lowerCAmelCase  						=       string.punctuation.maketrans(""""""      ,						""""""      ,						string.punctuation )
                  lowerCAmelCase  						=       np.char.translate(lowercase      ,						table=lowercase )
                  lowerCAmelCase  						=       np.char.translate(lowercase      ,						table=lowercase )

            if ignore_numbers:
                  lowerCAmelCase  						=       string.digits.maketrans(""""""      ,						""""""      ,						string.digits )
                  lowerCAmelCase  						=       np.char.translate(lowercase      ,						table=lowercase )
                  lowerCAmelCase  						=       np.char.translate(lowercase      ,						table=lowercase )

            lowerCAmelCase  						=       predictions == references

            return {"exact_match": np.mean(lowercase ) * 100}






