


"""simple docstring"""







import os
import re
import sys
import traceback
import warnings
from pathlib import Path
from typing import Dict, Optional, Union
from uuid import uuida

from huggingface_hub import HfFolder, ModelCard, ModelCardData, hf_hub_download, whoami
from huggingface_hub.file_download import REGEX_COMMIT_HASH
from huggingface_hub.utils import (
    EntryNotFoundError,
    RepositoryNotFoundError,
    RevisionNotFoundError,
    is_jinja_available,
)
from packaging import version
from requests import HTTPError

from .. import __version__
from .constants import (
    DEPRECATED_REVISION_ARGS,
    DIFFUSERS_CACHE,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    SAFETENSORS_WEIGHTS_NAME,
    WEIGHTS_NAME,
)
from .import_utils import (
    ENV_VARS_TRUE_VALUES,
    _flax_version,
    _jax_version,
    _onnxruntime_version,
    _torch_version,
    is_flax_available,
    is_onnx_available,
    is_torch_available,
)
from .logging import get_logger


a__				:							List[str]						=     get_logger(__name__)


a__				:							Optional[Any]						=     Path(__file__).parent / '''model_card_template.md'''
a__				:							Optional[int]						=     uuida().hex
a__				:							Any						=     os.getenv('''HF_HUB_OFFLINE''', '''''').upper() in ENV_VARS_TRUE_VALUES
a__				:							Any						=     os.getenv('''DISABLE_TELEMETRY''', '''''').upper() in ENV_VARS_TRUE_VALUES
a__				:							int						=     HUGGINGFACE_CO_RESOLVE_ENDPOINT + '''/api/telemetry/'''




def    UpperCAmelCase__       (lowerCAmelCase_ = None   ):




					'''simple docstring'''


					__SCREAMING_SNAKE_CASE						   =				f"""diffusers/{__version__}; python/{sys.version.split()[0]}; session_id/{SESSION_ID}"""
					if DISABLE_TELEMETRY or HF_HUB_OFFLINE:
										return ua + "; telemetry/off"
					if is_torch_available():
										ua += f"""; torch/{_torch_version}"""
					if is_flax_available():
										ua += f"""; jax/{_jax_version}"""
										ua += f"""; flax/{_flax_version}"""
					if is_onnx_available():
										ua += f"""; onnxruntime/{_onnxruntime_version}"""
					# CI will set this value to True
					if os.environ.get("DIFFUSERS_IS_CI"  ,						""   ).upper() in ENV_VARS_TRUE_VALUES:
										ua += "; is_ci/true"
					if isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ):
										ua += "; " + "; ".join(f"""{k}/{v}""" for k, v in user_agent.items()   )
					elif isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ):
										ua += "; " + user_agent
					return ua




def    UpperCAmelCase__       (lowerCAmelCase_  ,						lowerCAmelCase_ = None  ,						lowerCAmelCase_ = None   ):




					'''simple docstring'''


					if token is None:
										__SCREAMING_SNAKE_CASE						   =				HfFolder.get_token()
					if organization is None:
										__SCREAMING_SNAKE_CASE						   =				whoami(lowerCAmelCase_   )["name"]
										return f"""{username}/{model_id}"""
					else:
										return f"""{organization}/{model_id}"""




def    UpperCAmelCase__       (lowerCAmelCase_  ,						lowerCAmelCase_   ):




					'''simple docstring'''


					if not is_jinja_available():
										raise ValueError(
										    "Modelcard rendering is based on Jinja templates."
										    " Please make sure to have `jinja` installed before using `create_model_card`."
										    " To install it, please run `pip install Jinja2`."   )

					if hasattr(lowerCAmelCase_  ,						"local_rank"   ) and args.local_rank not in [-1, 0]:
										return

					__SCREAMING_SNAKE_CASE						   =				args.hub_token if hasattr(lowerCAmelCase_  ,						"hub_token"   ) else None
					__SCREAMING_SNAKE_CASE						   =				get_full_repo_name(lowerCAmelCase_  ,						token=lowerCAmelCase_   )

					__SCREAMING_SNAKE_CASE						   =				ModelCard.from_template(
					    card_data=ModelCardData(  # Card metadata object that will be converted to YAML block
					        language="en"  ,						license="apache-2.0"  ,						library_name="diffusers"  ,						tags=[]  ,						datasets=args.dataset_name  ,						metrics=[]  ,						)  ,						template_path=lowerCAmelCase_  ,						model_name=lowerCAmelCase_  ,						repo_name=lowerCAmelCase_  ,						dataset_name=args.dataset_name if hasattr(lowerCAmelCase_  ,						"dataset_name"   ) else None  ,						learning_rate=args.learning_rate  ,						train_batch_size=args.train_batch_size  ,						eval_batch_size=args.eval_batch_size  ,						gradient_accumulation_steps=(
					        args.gradient_accumulation_steps if hasattr(lowerCAmelCase_  ,						"gradient_accumulation_steps"   ) else None
					    )  ,						adam_betaa=args.adam_betaa if hasattr(lowerCAmelCase_  ,						"adam_beta1"   ) else None  ,						adam_betaa=args.adam_betaa if hasattr(lowerCAmelCase_  ,						"adam_beta2"   ) else None  ,						adam_weight_decay=args.adam_weight_decay if hasattr(lowerCAmelCase_  ,						"adam_weight_decay"   ) else None  ,						adam_epsilon=args.adam_epsilon if hasattr(lowerCAmelCase_  ,						"adam_epsilon"   ) else None  ,						lr_scheduler=args.lr_scheduler if hasattr(lowerCAmelCase_  ,						"lr_scheduler"   ) else None  ,						lr_warmup_steps=args.lr_warmup_steps if hasattr(lowerCAmelCase_  ,						"lr_warmup_steps"   ) else None  ,						ema_inv_gamma=args.ema_inv_gamma if hasattr(lowerCAmelCase_  ,						"ema_inv_gamma"   ) else None  ,						ema_power=args.ema_power if hasattr(lowerCAmelCase_  ,						"ema_power"   ) else None  ,						ema_max_decay=args.ema_max_decay if hasattr(lowerCAmelCase_  ,						"ema_max_decay"   ) else None  ,						mixed_precision=args.mixed_precision  ,						)

					__SCREAMING_SNAKE_CASE						   =				os.path.join(args.output_dir  ,						"README.md"   )
					model_card.save(lowerCAmelCase_   )




def    UpperCAmelCase__       (lowerCAmelCase_  ,						lowerCAmelCase_ = None   ):




					'''simple docstring'''


					if resolved_file is None or commit_hash is not None:
										return commit_hash
					__SCREAMING_SNAKE_CASE						   =				str(Path(lowerCAmelCase_   ).as_posix()   )
					__SCREAMING_SNAKE_CASE						   =				re.search(R"snapshots/([^/]+)/"  ,						lowerCAmelCase_   )
					if search is None:
										return None
					__SCREAMING_SNAKE_CASE						   =				search.groups()[0]
					return commit_hash if REGEX_COMMIT_HASH.match(lowerCAmelCase_   ) else None


# Old default cache path, potentially to be migrated.
# This logic was more or less taken from `transformers`, with the following differences:
# - Diffusers doesn't use custom environment variables to specify the cache path.
# - There is no need to migrate the cache format, just move the files to the new location.
a__				:							Tuple						=     os.path.expanduser(
    os.getenv('''HF_HOME''', os.path.join(os.getenv('''XDG_CACHE_HOME''', '''~/.cache'''), '''huggingface'''))
)
a__				:							int						=     os.path.join(hf_cache_home, '''diffusers''')




def    UpperCAmelCase__       (lowerCAmelCase_ = None  ,						lowerCAmelCase_ = None   ):




					'''simple docstring'''


					if new_cache_dir is None:
										__SCREAMING_SNAKE_CASE						   =				DIFFUSERS_CACHE
					if old_cache_dir is None:
										__SCREAMING_SNAKE_CASE						   =				old_diffusers_cache

					__SCREAMING_SNAKE_CASE						   =				Path(lowerCAmelCase_   ).expanduser()
					__SCREAMING_SNAKE_CASE						   =				Path(lowerCAmelCase_   ).expanduser()
					for old_blob_path in old_cache_dir.glob("**/blobs/*"   ):
										if old_blob_path.is_file() and not old_blob_path.is_symlink():
															__SCREAMING_SNAKE_CASE						   =				new_cache_dir / old_blob_path.relative_to(lowerCAmelCase_   )
															new_blob_path.parent.mkdir(parents=lowerCAmelCase_  ,						exist_ok=lowerCAmelCase_   )
															os.replace(lowerCAmelCase_  ,						lowerCAmelCase_   )
															try:
																				os.symlink(lowerCAmelCase_  ,						lowerCAmelCase_   )
															except OSError:
																				logger.warning(
																				    "Could not create symlink between old cache and new cache. If you use an older version of diffusers again, files will be re-downloaded."   )
					# At this point, old_cache_dir contains symlinks to the new cache (it can still be used).


a__				:							int						=     os.path.join(DIFFUSERS_CACHE, '''version_diffusers_cache.txt''')
if not os.path.isfile(cache_version_file):
		a__				:							Any						=     0
else:
		with open(cache_version_file) as f:
				try:
						a__				:							Optional[int]						=     int(f.read())
				except ValueError:
						a__				:							Optional[int]						=     0

if cache_version < 1:
		a__				:							List[Any]						=     os.path.isdir(old_diffusers_cache) and len(os.listdir(old_diffusers_cache)) > 0
		if old_cache_is_not_empty:
				logger.warning(
				    '''The cache for model files in Diffusers v0.14.0 has moved to a new location. Moving your '''
				    '''existing cached models. This is a one-time operation, you can interrupt it or run it '''
				    '''later by calling `diffusers.utils.hub_utils.move_cache()`.'''
				)
				try:
						move_cache()
				except Exception as e:
						a__				:							Union[str, Any]						=     '''\n'''.join(traceback.format_tb(e.__traceback__))
						logger.error(
						    F"There was a problem when trying to move your cache:\n\n{trace}\n{e.__class__.__name__}: {e}\n\nPlease "
						    '''file an issue at https://github.com/huggingface/diffusers/issues/new/choose, copy paste this whole '''
						    '''message and we will do our best to help.'''
						)

if cache_version < 1:
		try:
				os.makedirs(DIFFUSERS_CACHE, exist_ok=True)
				with open(cache_version_file, '''w''') as f:
						f.write('''1''')
		except Exception:
				logger.warning(
				    F"There was a problem when trying to write in your cache folder ({DIFFUSERS_CACHE}). Please, ensure "
				    '''the directory exists and can be written to.'''
				)




def    UpperCAmelCase__       (lowerCAmelCase_  ,						lowerCAmelCase_ = None   ):




					'''simple docstring'''


					if variant is not None:
										__SCREAMING_SNAKE_CASE						   =				weights_name.split("."   )
										__SCREAMING_SNAKE_CASE						   =				splits[:-1] + [variant] + splits[-1:]
										__SCREAMING_SNAKE_CASE						   =				".".join(lowerCAmelCase_   )

					return weights_name




def    UpperCAmelCase__       (lowerCAmelCase_  ,						*,
    lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_=None  ,						):




					'''simple docstring'''


					__SCREAMING_SNAKE_CASE						   =				str(lowerCAmelCase_   )
					if os.path.isfile(lowerCAmelCase_   ):
										return pretrained_model_name_or_path
					elif os.path.isdir(lowerCAmelCase_   ):
										if os.path.isfile(os.path.join(lowerCAmelCase_  ,						lowerCAmelCase_   )   ):
															# Load from a PyTorch checkpoint
															__SCREAMING_SNAKE_CASE						   =				os.path.join(lowerCAmelCase_  ,						lowerCAmelCase_   )
															return model_file
										elif subfolder is not None and os.path.isfile(
										    os.path.join(lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_   )   ):
															__SCREAMING_SNAKE_CASE						   =				os.path.join(lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_   )
															return model_file
										else:
															raise EnvironmentError(
															    f"""Error no file named {weights_name} found in directory {pretrained_model_name_or_path}."""   )
					else:
										# 1. First check if deprecated way of loading from branches is used
										if (
										    revision in DEPRECATED_REVISION_ARGS
										    and (weights_name == WEIGHTS_NAME or weights_name == SAFETENSORS_WEIGHTS_NAME)
										    and version.parse(version.parse(lowerCAmelCase_   ).base_version   ) >= version.parse("0.20.0"   )
										):
															try:
																				__SCREAMING_SNAKE_CASE						   =				hf_hub_download(
																				    lowerCAmelCase_  ,						filename=_add_variant(lowerCAmelCase_  ,						lowerCAmelCase_   )  ,						cache_dir=lowerCAmelCase_  ,						force_download=lowerCAmelCase_  ,						proxies=lowerCAmelCase_  ,						resume_download=lowerCAmelCase_  ,						local_files_only=lowerCAmelCase_  ,						use_auth_token=lowerCAmelCase_  ,						user_agent=lowerCAmelCase_  ,						subfolder=lowerCAmelCase_  ,						revision=revision or commit_hash  ,						)
																				warnings.warn(
																				    f"""Loading the variant {revision} from {pretrained_model_name_or_path} via `revision='{revision}'` is deprecated. Loading instead from `revision='main'` with `variant={revision}`. Loading model variants via `revision='{revision}'` will be removed in diffusers v1. Please use `variant='{revision}'` instead."""  ,						lowerCAmelCase_  ,						)
																				return model_file
															except:  # noqa: E722
																				warnings.warn(
																				    f"""You are loading the variant {revision} from {pretrained_model_name_or_path} via `revision='{revision}'`. This behavior is deprecated and will be removed in diffusers v1. One should use `variant='{revision}'` instead. However, it appears that {pretrained_model_name_or_path} currently does not have a {_add_variant(lowerCAmelCase_  ,						lowerCAmelCase_   )} file in the 'main' branch of {pretrained_model_name_or_path}. \n The Diffusers team and community would be very grateful if you could open an issue: https://github.com/huggingface/diffusers/issues/new with the title '{pretrained_model_name_or_path} is missing {_add_variant(lowerCAmelCase_  ,						lowerCAmelCase_   )}' so that the correct variant file can be added."""  ,						lowerCAmelCase_  ,						)
										try:
															# 2. Load model file as usual
															__SCREAMING_SNAKE_CASE						   =				hf_hub_download(
															    lowerCAmelCase_  ,						filename=lowerCAmelCase_  ,						cache_dir=lowerCAmelCase_  ,						force_download=lowerCAmelCase_  ,						proxies=lowerCAmelCase_  ,						resume_download=lowerCAmelCase_  ,						local_files_only=lowerCAmelCase_  ,						use_auth_token=lowerCAmelCase_  ,						user_agent=lowerCAmelCase_  ,						subfolder=lowerCAmelCase_  ,						revision=revision or commit_hash  ,						)
															return model_file

										except RepositoryNotFoundError:
															raise EnvironmentError(
															    f"""{pretrained_model_name_or_path} is not a local folder and is not a valid model identifier """
															    "listed on 'https://huggingface.co/models'\nIf this is a private repository, make sure to pass a "
															    "token having permission to this repo with `use_auth_token` or log in with `huggingface-cli "
															    "login`."   )
										except RevisionNotFoundError:
															raise EnvironmentError(
															    f"""{revision} is not a valid git identifier (branch name, tag name or commit id) that exists for """
															    "this model name. Check the model page at "
															    f"""'https://huggingface.co/{pretrained_model_name_or_path}' for available revisions."""   )
										except EntryNotFoundError:
															raise EnvironmentError(
															    f"""{pretrained_model_name_or_path} does not appear to have a file named {weights_name}."""   )
										except HTTPError as err:
															raise EnvironmentError(
															    f"""There was a specific connection error when trying to load {pretrained_model_name_or_path}:\n{err}"""   )
										except ValueError:
															raise EnvironmentError(
															    f"""We couldn't connect to '{HUGGINGFACE_CO_RESOLVE_ENDPOINT}' to load this model, couldn't find it"""
															    f""" in the cached files and it looks like {pretrained_model_name_or_path} is not the path to a"""
															    f""" directory containing a file named {weights_name} or"""
															    " \nCheckout your internet connection or see how to run the library in"
															    " offline mode at 'https://huggingface.co/docs/diffusers/installation#offline-mode'."   )
										except EnvironmentError:
															raise EnvironmentError(
															    f"""Can't load the model for '{pretrained_model_name_or_path}'. If you were trying to load it from """
															    "'https://huggingface.co/models', make sure you don't have a local directory with the same name. "
															    f"""Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a directory """
															    f"""containing a file named {weights_name}"""   )





'''simple docstring'''





from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


_lowerCAmelCase      						=	'''\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
'''

_lowerCAmelCase      						=	'''\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
'''

_lowerCAmelCase      						=	'''
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for \'record\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'prediction_text\': the predicted answer text
        - for \'multirc\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question-answer pair as specified by the dataset
            - \'prediction\': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for \'record\': list of question-answers dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'answers\': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for \'record\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1\': F1 score
    - for \'multirc\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1_m\': Per-question macro-F1 score
        - \'f1_a\': Average F1 score over all answers
    - for \'axb\':
        \'matthews_correlation\': Matthew Correlation
    - for \'cb\':
        - \'accuracy\': Accuracy
        - \'f1\': F1 score
    - for all others:
        - \'accuracy\': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')
    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]
    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')
    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'matthews_correlation\': 1.0}
'''
def    _SCREAMING_SNAKE_CASE  (       UpperCamelCase							,    UpperCamelCase			):



    """simple docstring"""





    return float((preds == labels).mean()			)
def    _SCREAMING_SNAKE_CASE  (       UpperCamelCase							,    UpperCamelCase							,    UpperCamelCase="binary"			):



    """simple docstring"""





    lowerCAmelCase__ :  Any							      =					simple_accuracy(UpperCamelCase							,    UpperCamelCase			)
    lowerCAmelCase__ :  Tuple							      =					float(fa_score(y_true=UpperCamelCase							,    y_pred=UpperCamelCase							,    average=UpperCamelCase			)			)
    return {
        "accuracy": acc,
        "f1": fa,
    }
def    _SCREAMING_SNAKE_CASE  (       UpperCamelCase							,    UpperCamelCase			):



    """simple docstring"""





    lowerCAmelCase__ :  List[str]							      =					{}
    for id_pred, label in zip(UpperCamelCase							,    UpperCamelCase			):
        lowerCAmelCase__ :  str							      =					f"""{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}"""
        lowerCAmelCase__ :  Dict							      =					id_pred["""prediction"""]
        if question_id in question_map:
            question_map[question_id].append((pred, label)			)
        else:
            lowerCAmelCase__ :  Optional[int]							      =					[(pred, label)]
    lowerCAmelCase__      ,   lowerCAmelCase__ :  int							      =					[], []
    for question, preds_labels in question_map.items():
        lowerCAmelCase__      ,   lowerCAmelCase__ :  Optional[int]							      =					zip(*UpperCamelCase			)
        lowerCAmelCase__ :  List[Any]							      =					fa_score(y_true=UpperCamelCase							,    y_pred=UpperCamelCase							,    average="""macro"""			)
        fas.append(UpperCamelCase			)
        lowerCAmelCase__ :  Union[str, Any]							      =					int(sum(pred == label for pred, label in preds_labels			) == len(UpperCamelCase			)			)
        ems.append(UpperCamelCase			)
    lowerCAmelCase__ :  Optional[Any]							      =					float(sum(UpperCamelCase			) / len(UpperCamelCase			)			)
    lowerCAmelCase__ :  List[Any]							      =					sum(UpperCamelCase			) / len(UpperCamelCase			)
    lowerCAmelCase__ :  Dict							      =					float(fa_score(y_true=UpperCamelCase							,    y_pred=[id_pred["""prediction"""] for id_pred in ids_preds]			)			)
    return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION						, _KWARGS_DESCRIPTION     )
class        lowerCAmelCase_(				datasets.Metric     ):
  '''simple docstring'''




  def 							UpperCAmelCase_	(    self      )    ->		Optional[Any]:
      if self.config_name not in [
          "boolq",
          "cb",
          "copa",
          "multirc",
          "record",
          "rte",
          "wic",
          "wsc",
          "wsc.fixed",
          "axb",
          "axg",
      ]:
          raise KeyError(
              """You should supply a configuration name selected in """
              """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"""      )
      return datasets.MetricInfo(
          description=_DESCRIPTION  ,citation=_CITATION  ,inputs_description=_KWARGS_DESCRIPTION  ,features=datasets.Features(self._get_feature_types()      )  ,codebase_urls=[]  ,reference_urls=[]  ,format="""numpy""" if not self.config_name == """record""" and not self.config_name == """multirc""" else None  ,)




  def 							UpperCAmelCase_	(    self      )    ->		str:
      if self.config_name == "record":
          return {
              "predictions": {
                  "idx": {
                      "passage": datasets.Value("""int64"""      ),
                      "query": datasets.Value("""int64"""      ),
                  },
                  "prediction_text": datasets.Value("""string"""      ),
              },
              "references": {
                  "idx": {
                      "passage": datasets.Value("""int64"""      ),
                      "query": datasets.Value("""int64"""      ),
                  },
                  "answers": datasets.Sequence(datasets.Value("""string"""      )      ),
              },
          }
      elif self.config_name == "multirc":
          return {
              "predictions": {
                  "idx": {
                      "answer": datasets.Value("""int64"""      ),
                      "paragraph": datasets.Value("""int64"""      ),
                      "question": datasets.Value("""int64"""      ),
                  },
                  "prediction": datasets.Value("""int64"""      ),
              },
              "references": datasets.Value("""int64"""      ),
          }
      else:
          return {
              "predictions": datasets.Value("""int64"""      ),
              "references": datasets.Value("""int64"""      ),
          }






  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase      )    ->		Any:
      if self.config_name == "axb":
          return {"matthews_correlation": matthews_corrcoef(__UpperCAmelCase  ,__UpperCAmelCase      )}
      elif self.config_name == "cb":
          return acc_and_fa(__UpperCAmelCase  ,__UpperCAmelCase  ,fa_avg="""macro"""      )
      elif self.config_name == "record":
          lowerCAmelCase__ :  Optional[Any]							      =					[
              {
                  """qas""": [
                      {"""id""": ref["""idx"""]["""query"""], """answers""": [{"""text""": ans} for ans in ref["""answers"""]]}
                      for ref in references
                  ]
              }
          ]
          lowerCAmelCase__ :  Union[str, Any]							      =					{pred["""idx"""]["""query"""]: pred["""prediction_text"""] for pred in predictions}
          return evaluate_record(__UpperCAmelCase  ,__UpperCAmelCase      )[0]
      elif self.config_name == "multirc":
          return evaluate_multirc(__UpperCAmelCase  ,__UpperCAmelCase      )
      elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
          return {"accuracy": simple_accuracy(__UpperCAmelCase  ,__UpperCAmelCase      )}
      else:
          raise KeyError(
              """You should supply a configuration name selected in """
              """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"""      )


