"""simple docstring"""
from sklearn.metrics import roc_auc_score

import datasets


lowercase__   			=      '\nThis metric computes the area under the curve (AUC) for the Receiver Operating Characteristic Curve (ROC). The return values represent how well the model used is predicting the correct classes, based on the input data. A score of `0.5` means that the model is predicting exactly at chance, i.e. the model\'s predictions are correct at the same rate as if the predictions were being decided by the flip of a fair coin or the roll of a fair die. A score above `0.5` indicates that the model is doing better than chance, while a score below `0.5` indicates that the model is doing worse than chance.\n\nThis metric has three separate use cases:\n    - binary: The case in which there are only two different label classes, and each example gets only one label. This is the default implementation.\n    - multiclass: The case in which there can be more than two different label classes, but each example still gets only one label.\n    - multilabel: The case in which there can be more than two different label classes, and each example can have more than one label.\n'

lowercase__   			=      '\nArgs:\n- references (array-like of shape (n_samples,) or (n_samples, n_classes)): Ground truth labels. Expects different input based on use case:\n    - binary: expects an array-like of shape (n_samples,)\n    - multiclass: expects an array-like of shape (n_samples,)\n    - multilabel: expects an array-like of shape (n_samples, n_classes)\n- prediction_scores (array-like of shape (n_samples,) or (n_samples, n_classes)): Model predictions. Expects different inputs based on use case:\n    - binary: expects an array-like of shape (n_samples,)\n    - multiclass: expects an array-like of shape (n_samples, n_classes)\n    - multilabel: expects an array-like of shape (n_samples, n_classes)\n- average (`str`): Type of average, and is ignored in the binary use case. Defaults to \'macro\'. Options are:\n    - `\'micro\'`: Calculates metrics globally by considering each element of the label indicator matrix as a label. Only works with the multilabel use case.\n    - `\'macro\'`: Calculate metrics for each label, and find their unweighted mean.  This does not take label imbalance into account.\n    - `\'weighted\'`: Calculate metrics for each label, and find their average, weighted by support (i.e. the number of true instances for each label).\n    - `\'samples\'`: Calculate metrics for each instance, and find their average. Only works with the multilabel use case.\n    - `None`:  No average is calculated, and scores for each class are returned. Only works with the multilabels use case.\n- sample_weight (array-like of shape (n_samples,)): Sample weights. Defaults to None.\n- max_fpr (`float`): If not None, the standardized partial AUC over the range [0, `max_fpr`] is returned. Must be greater than `0` and less than or equal to `1`. Defaults to `None`. Note: For the multiclass use case, `max_fpr` should be either `None` or `1.0` as ROC AUC partial computation is not currently supported for `multiclass`.\n- multi_class (`str`): Only used for multiclass targets, where it is required. Determines the type of configuration to use. Options are:\n    - `\'ovr\'`: Stands for One-vs-rest. Computes the AUC of each class against the rest. This treats the multiclass case in the same way as the multilabel case. Sensitive to class imbalance even when `average == \'macro\'`, because class imbalance affects the composition of each of the \'rest\' groupings.\n    - `\'ovo\'`: Stands for One-vs-one. Computes the average AUC of all possible pairwise combinations of classes. Insensitive to class imbalance when `average == \'macro\'`.\n- labels (array-like of shape (n_classes,)): Only used for multiclass targets. List of labels that index the classes in\n    `prediction_scores`. If `None`, the numerical or lexicographical order of the labels in\n    `prediction_scores` is used. Defaults to `None`.\nReturns:\n    roc_auc (`float` or array-like of shape (n_classes,)): Returns array if in multilabel use case and `average=\'None\'`. Otherwise, returns `float`.\nExamples:\n    Example 1:\n        >>> roc_auc_score = datasets.load_metric("roc_auc")\n        >>> refs = [1, 0, 1, 1, 0, 0]\n        >>> pred_scores = [0.5, 0.2, 0.99, 0.3, 0.1, 0.7]\n        >>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores)\n        >>> print(round(results[\'roc_auc\'], 2))\n        0.78\n\n    Example 2:\n        >>> roc_auc_score = datasets.load_metric("roc_auc", "multiclass")\n        >>> refs = [1, 0, 1, 2, 2, 0]\n        >>> pred_scores = [[0.3, 0.5, 0.2],\n        ...                 [0.7, 0.2, 0.1],\n        ...                 [0.005, 0.99, 0.005],\n        ...                 [0.2, 0.3, 0.5],\n        ...                 [0.1, 0.1, 0.8],\n        ...                 [0.1, 0.7, 0.2]]\n        >>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores, multi_class=\'ovr\')\n        >>> print(round(results[\'roc_auc\'], 2))\n        0.85\n\n    Example 3:\n        >>> roc_auc_score = datasets.load_metric("roc_auc", "multilabel")\n        >>> refs = [[1, 1, 0],\n        ...         [1, 1, 0],\n        ...         [0, 1, 0],\n        ...         [0, 0, 1],\n        ...         [0, 1, 1],\n        ...         [1, 0, 1]]\n        >>> pred_scores = [[0.3, 0.5, 0.2],\n        ...                 [0.7, 0.2, 0.1],\n        ...                 [0.005, 0.99, 0.005],\n        ...                 [0.2, 0.3, 0.5],\n        ...                 [0.1, 0.1, 0.8],\n        ...                 [0.1, 0.7, 0.2]]\n        >>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores, average=None)\n        >>> print([round(res, 2) for res in results[\'roc_auc\']])\n        [0.83, 0.38, 0.94]\n'

lowercase__   			=      '\\n@article{doi:10.1177/0272989X8900900307,\nauthor = {Donna Katzman McClish},\ntitle ={Analyzing a Portion of the ROC Curve},\njournal = {Medical Decision Making},\nvolume = {9},\nnumber = {3},\npages = {190-195},\nyear = {1989},\ndoi = {10.1177/0272989X8900900307},\n    note ={PMID: 2668680},\nURL = {https://doi.org/10.1177/0272989X8900900307},\neprint = {https://doi.org/10.1177/0272989X8900900307}\n}\n\n\n@article{10.1023/A:1010920819831,\nauthor = {Hand, David J. and Till, Robert J.},\ntitle = {A Simple Generalisation of the Area Under the ROC Curve for Multiple Class Classification Problems},\nyear = {2001},\nissue_date = {November 2001},\npublisher = {Kluwer Academic Publishers},\naddress = {USA},\nvolume = {45},\nnumber = {2},\nissn = {0885-6125},\nurl = {https://doi.org/10.1023/A:1010920819831},\ndoi = {10.1023/A:1010920819831},\njournal = {Mach. Learn.},\nmonth = {oct},\npages = {171–186},\nnumpages = {16},\nkeywords = {Gini index, AUC, error rate, ROC curve, receiver operating characteristic}\n}\n\n\n@article{scikit-learn,\ntitle={Scikit-learn: Machine Learning in {P}ython},\nauthor={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},\njournal={Journal of Machine Learning Research},\nvolume={12},\npages={2825--2830},\nyear={2011}\n}\n'




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION			, _KWARGS_DESCRIPTION							)
class   __snake_case			(	datasets.Metric							):



      def     lowerCamelCase_  (		self)      ->     Optional[Any]:





        '''simple docstring'''





        return datasets.MetricInfo(
            description=_DESCRIPTION       ,     citation=_CITATION       ,     inputs_description=_KWARGS_DESCRIPTION       ,     features=datasets.Features(
                {
                    'prediction_scores': datasets.Sequence(datasets.Value('float')),
                    'references': datasets.Value('int32'),
                }
                if self.config_name == 'multiclass'
                else {
                    'references': datasets.Sequence(datasets.Value('int32')),
                    'prediction_scores': datasets.Sequence(datasets.Value('float')),
                }
                if self.config_name == 'multilabel'
                else {
                    'references': datasets.Value('int32'),
                    'prediction_scores': datasets.Value('float'),
                })       ,     reference_urls=['https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc_auc_score.html']       ,     )



      def     lowerCamelCase_  (		self       ,     lowercase       ,     lowercase       ,     lowercase="macro"       ,     lowercase=None       ,     lowercase=None       ,     lowercase="raise"       ,     lowercase=None       ,     )      ->     Optional[int]:





        '''simple docstring'''





        return {
            "roc_auc": roc_auc_score(
                lowercase       ,     lowercase       ,     average=lowercase       ,     sample_weight=lowercase       ,     max_fpr=lowercase       ,     multi_class=lowercase       ,     labels=lowercase       ,     )
        }



"""simple docstring"""
import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
 import torch

lowercase__   			=      logging.get_logger(__name__)







def 		__a  (      )      ->Dict:
  return importlib.util.find_spec('deepspeed'  ) is not None


if is_accelerate_available() and is_deepspeed_available():
 from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
 # Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
 # Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
 from builtins import object as DeepSpeedConfig




class   __snake_case			(	__lowerCAmelCase							):



      def __init__(		self       ,     lowercase)      ->     int:





        '''simple docstring'''





        set_hf_deepspeed_config(self)
        dep_version_check('accelerate')
        dep_version_check('deepspeed')
        super().__init__(lowercase)







class   __snake_case			(	__lowerCAmelCase							):



      def __init__(		self       ,     lowercase)      ->     List[Any]:





        '''simple docstring'''





        super().__init__(lowercase)
        a__:	List[Any]     				=      None
        a__:	int     				=      []



      def     lowerCamelCase_  (		self)      ->     Any:





        '''simple docstring'''





        if self._dtype is None:
          raise ValueError('trainer_config_process() wasn\'t called yet to tell dtype')
        return self._dtype



      def     lowerCamelCase_  (		self       ,     lowercase)      ->     Optional[Any]:





        '''simple docstring'''





        a__:	Optional[int]     				=      self.get_value(lowercase)
        if val is None:
          return False
        else:
          return val == "auto"



      def     lowerCamelCase_  (		self       ,     lowercase       ,     lowercase       ,     lowercase=None       ,     lowercase=True)      ->     int:





        '''simple docstring'''





        a__				,       a__:	int     				=      self.find_config_node(lowercase)
        if config is None:
          return

        if config.get(lowercase) == "auto":
          a__:	Optional[int]     				=      hf_val
          return

        if not must_match:
          return

        a__:	List[Any]     				=      config.get(lowercase)
        if ds_val is not None and ds_val != hf_val:
          self.mismatches.append(f'- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}')

      a__								=						partialmethod(__lowerCAmelCase			, must_match=__lowerCAmelCase							)



      def     lowerCamelCase_  (		self       ,     lowercase)      ->     Dict:





        '''simple docstring'''





        a__:	str     				=      args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
        self.fill_match(
            'train_micro_batch_size_per_gpu'       ,     args.per_device_train_batch_size       ,     'per_device_train_batch_size')
        self.fill_match('gradient_accumulation_steps'       ,     args.gradient_accumulation_steps       ,     'gradient_accumulation_steps')
        self.fill_match('train_batch_size'       ,     lowercase       ,     'train_batch_size (calculated)')
        self.fill_match('gradient_clipping'       ,     args.max_grad_norm       ,     'max_grad_norm')

        self.fill_match('optimizer.params.lr'       ,     args.learning_rate       ,     'learning_rate')
        self.fill_match('optimizer.params.betas'       ,     [args.adam_betaa, args.adam_betaa]       ,     'adam_beta1+adam_beta2')
        self.fill_match('optimizer.params.eps'       ,     args.adam_epsilon       ,     'adam_epsilon')
        self.fill_match('optimizer.params.weight_decay'       ,     args.weight_decay       ,     'weight_decay')

        self.fill_only('scheduler.params.warmup_min_lr'       ,     0)  # not a trainer arg
        self.fill_match('scheduler.params.warmup_max_lr'       ,     args.learning_rate       ,     'learning_rate')
        # total_num_steps - will get set in trainer_config_finalize

        # fp16
        if args.fpaa or args.fpaa_full_eval:
          a__:	Tuple     				=      'apex' if args.fpaa_backend == 'apex' else 'amp'
        else:
          a__:	Optional[int]     				=      None

        if args.save_on_each_node:
          # deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
          a__:	Union[str, Any]     				=      self.config.get('checkpoint'       ,     {})
          a__:	Tuple     				=      args.save_on_each_node

        # amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
        # any here unless the user did the work
        self.fill_match(
            'fp16.enabled'       ,     ((args.fpaa or args.fpaa_full_eval) and fpaa_backend == 'amp')       ,     'fp16|fp16_full_eval+fp16_backend(amp)'       ,     )

        # apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
        # ZeRO features
        self.fill_match('amp.enabled'       ,     fpaa_backend == 'apex'       ,     'fp16+fp16_backend(apex)')
        self.fill_match('amp.opt_level'       ,     args.fpaa_opt_level       ,     'fp16_opt_level')

        self.fill_match('bf16.enabled'       ,     (args.bfaa or args.bfaa_full_eval)       ,     'bf16|bf16_full_eval')

        # deepspeed's default mode is fp16 unless there is a config that says differently
        if self.is_true('bf16.enabled'):
          a__:	Optional[int]     				=      torch.bfloataa
        elif self.is_false('fp16.enabled'):
          a__:	Tuple     				=      torch.floataa
        else:
          a__:	Tuple     				=      torch.floataa



      def     lowerCamelCase_  (		self       ,     lowercase       ,     lowercase       ,     lowercase)      ->     Any:





        '''simple docstring'''





        a__:	List[Any]     				=      [
            'zero_optimization.reduce_bucket_size',
            'zero_optimization.stage3_prefetch_bucket_size',
            'zero_optimization.stage3_param_persistence_threshold',
        ]
        a__:	Union[str, Any]     				=      [x for x in hidden_size_based_keys if self.is_auto(lowercase)]

        if len(lowercase) > 0:
          if hasattr(model.config       ,     'hidden_size'):
            a__:	Any     				=      model.config.hidden_size
          elif hasattr(model.config       ,     'hidden_sizes'):
            # if there are many hidden sizes pick the largest one
            a__:	List[str]     				=      max(model.config.hidden_sizes)
          else:
            raise ValueError(
                'The model\'s config file has neither `hidden_size` nor `hidden_sizes` entry, '
                'therefore it\'s not possible to automatically fill out the following `auto` entries '
                f'in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing '
                '`auto` values for these keys with an integer value of your choice.')

          self.fill_only('zero_optimization.reduce_bucket_size'       ,     hidden_size * hidden_size)
          if self.is_zeroa():
            # automatically assign the optimal config values based on model config
            self.fill_only('zero_optimization.stage3_prefetch_bucket_size'       ,     0.9 * hidden_size * hidden_size)
            self.fill_only('zero_optimization.stage3_param_persistence_threshold'       ,     10 * hidden_size)

        # scheduler
        self.fill_match('scheduler.params.total_num_steps'       ,     lowercase       ,     'num_training_steps (calculated)')
        self.fill_match('scheduler.params.warmup_num_steps'       ,     args.get_warmup_steps(lowercase)       ,     'warmup_steps')

        if len(self.mismatches) > 0:
          a__:	int     				=      '\n'.join(self.mismatches)
          raise ValueError(
              'Please correct the following DeepSpeed config values that mismatch TrainingArguments'
              f' values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to \'auto\'.')







lowercase__   			=      None







def 		__a  (      _SCREAMING_SNAKE_CASE  )      ->List[Any]:
  # this is a special weakref global object to allow us to get to Deepspeed config from APIs
  # that don't have an easy way to get to the Deepspeed config outside of the Trainer domain.
  global _hf_deepspeed_config_weak_ref
  # will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
  a__:	Any     				=      weakref.ref(_SCREAMING_SNAKE_CASE  )







def 		__a  (      )      ->List[Any]:
  # useful for unit tests to ensure the global state doesn't leak - call from `tearDown` method
  global _hf_deepspeed_config_weak_ref
  a__:	Any     				=      None







def 		__a  (      )      ->Union[str, Any]:
  if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
    return _hf_deepspeed_config_weak_ref().is_zeroa()
  else:
    return False







def 		__a  (      )      ->Tuple:
  if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
    return _hf_deepspeed_config_weak_ref().config
  else:
    return None







def 		__a  (      _SCREAMING_SNAKE_CASE						,	_SCREAMING_SNAKE_CASE						,	_SCREAMING_SNAKE_CASE						,	_SCREAMING_SNAKE_CASE						,	_SCREAMING_SNAKE_CASE  )      ->Dict:
  from accelerate.utils import DummyOptim, DummyScheduler

  a__:	Optional[Any]     				=      hf_deepspeed_config.config

  # Optimizer + Scheduler
  # Currently supported combos:
  # 1. DS scheduler + DS optimizer: Yes
  # 2. HF scheduler + HF optimizer: Yes
  # 3. DS scheduler + HF optimizer: Yes
  # 4. HF scheduler + DS optimizer: No
  #
  # Unless Offload is enabled in which case it's:
  # 1. DS scheduler + DS optimizer: Yes
  # 2. HF scheduler + HF optimizer: Mostly*
  # 3. DS scheduler + HF optimizer: Mostly*
  # 4. HF scheduler + DS optimizer: No
  #
  # Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

  a__:	Dict     				=      None
  if "optimizer" in config:
    if args.adafactor:
      raise ValueError(
          '--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. '
          'Only one optimizer can be configured.'  )
    a__:	str     				=      DummyOptim(params=_SCREAMING_SNAKE_CASE  )
  else:
    if hf_deepspeed_config.is_offload():
      logger.info(
          'Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the'
          ' custom optimizer has both CPU and GPU implementation (except LAMB)'  )

    # ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
    # But trainer uses AdamW by default.
    a__:	Any     				=      trainer.create_optimizer()
    # To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
    a__:	str     				=      True

  a__:	Dict     				=      None
  if "scheduler" in config:
    a__:	List[str]     				=      DummyScheduler(_SCREAMING_SNAKE_CASE  )
  else:
    if isinstance(_SCREAMING_SNAKE_CASE						,	_SCREAMING_SNAKE_CASE  ):
      raise ValueError(
          'Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. '
          'Please configure a scheduler in the DeepSpeed config.'  )
    a__:	Tuple     				=      trainer.create_scheduler(num_training_steps=_SCREAMING_SNAKE_CASE						,	optimizer=_SCREAMING_SNAKE_CASE  )

  return optimizer, lr_scheduler







def 		__a  (      _SCREAMING_SNAKE_CASE						,	_SCREAMING_SNAKE_CASE						,	_SCREAMING_SNAKE_CASE=False  )      ->Dict:
  from deepspeed.utils import logger as ds_logger

  a__:	Optional[int]     				=      trainer.model
  a__:	List[Any]     				=      trainer.args

  a__:	Optional[int]     				=      trainer.accelerator.state.deepspeed_plugin.hf_ds_config

  # resume config update - some bits like `model` and `num_training_steps` only become available during train
  hf_deepspeed_config.trainer_config_finalize(_SCREAMING_SNAKE_CASE						,	_SCREAMING_SNAKE_CASE						,	_SCREAMING_SNAKE_CASE  )

  # set the Deepspeed log level consistent with the Trainer
  ds_logger.setLevel(args.get_process_log_level()  )

  if inference:
    # only Z3 makes sense for the inference
    if not hf_deepspeed_config.is_zeroa():
      raise ValueError('ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config'  )

    # in case the training config is re-used for inference
    hf_deepspeed_config.del_config_sub_tree('optimizer'  )
    hf_deepspeed_config.del_config_sub_tree('lr_scheduler'  )
    a__				,       a__:	Optional[int]     				=      None, None
    a__:	Union[str, Any]     				=      None
  else:
    a__:	List[Any]     				=      None  # important for when deepspeed_init is used as re-init
    a__:	Union[str, Any]     				=      list(filter(lambda _SCREAMING_SNAKE_CASE  : p.requires_grad						,	model.parameters()  )  )
    a__				,       a__:	Optional[int]     				=      deepspeed_optim_sched(
        _SCREAMING_SNAKE_CASE						,	_SCREAMING_SNAKE_CASE						,	_SCREAMING_SNAKE_CASE						,	_SCREAMING_SNAKE_CASE						,	_SCREAMING_SNAKE_CASE  )

  # keep for quick debug:
  # from pprint import pprint; pprint(config)

  return optimizer, lr_scheduler







def 		__a  (      _SCREAMING_SNAKE_CASE						,	_SCREAMING_SNAKE_CASE  )      ->int:
  # it's possible that the user is trying to resume from model_path, which doesn't necessarily
  # contain a deepspeed checkpoint. e.g. examples just check if the dir exists and assume it's
  # a resume from a checkpoint and not just a local pretrained weight. So we check here if the
  # path contains what looks like a deepspeed checkpoint
  import glob

  a__:	Tuple     				=      sorted(glob.glob(F'{checkpoint_path}/global_step*'  )  )

  if len(_SCREAMING_SNAKE_CASE  ) > 0:
    logger.info(F'Attempting to resume from {checkpoint_path}'  )
    # this magically updates self.optimizer and self.lr_scheduler
    a__				,       a__:	Union[str, Any]     				=      deepspeed_engine.load_checkpoint(
        _SCREAMING_SNAKE_CASE						,	load_optimizer_states=_SCREAMING_SNAKE_CASE						,	load_lr_scheduler_states=_SCREAMING_SNAKE_CASE  )
    if load_path is None:
      raise ValueError(F'[deepspeed] failed to resume from checkpoint {checkpoint_path}'  )
  else:
    raise ValueError(F'Can\'t find a valid checkpoint at {checkpoint_path}'  )


