


'''simple docstring'''







from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


A__:    Optional[int]					    =       '''\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n'''

A__:    Any					    =       '''\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n'''

A__:    str					    =       '''\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for \'record\': list of question-answer dictionaries with the following keys:\n            - \'idx\': index of the question as specified by the dataset\n            - \'prediction_text\': the predicted answer text\n        - for \'multirc\': list of question-answer dictionaries with the following keys:\n            - \'idx\': index of the question-answer pair as specified by the dataset\n            - \'prediction\': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for \'record\': list of question-answers dictionaries with the following keys:\n            - \'idx\': index of the question as specified by the dataset\n            - \'answers\': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for \'record\':\n        - \'exact_match\': Exact match between answer and gold answer\n        - \'f1\': F1 score\n    - for \'multirc\':\n        - \'exact_match\': Exact match between answer and gold answer\n        - \'f1_m\': Per-question macro-F1 score\n        - \'f1_a\': Average F1 score over all answers\n    - for \'axb\':\n        \'matthews_correlation\': Matthew Correlation\n    - for \'cb\':\n        - \'accuracy\': Accuracy\n        - \'f1\': F1 score\n    - for all others:\n        - \'accuracy\': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0, \'f1\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')\n    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]\n    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 1.0, \'f1\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')\n    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'matthews_correlation\': 1.0}\n'''


def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	int    ,_UpperCAmelCase	:	Dict  )  ->			Optional[Any]:
       return float((preds == labels).mean()  )


def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	Tuple    ,_UpperCAmelCase	:	Union[str, Any]    ,_UpperCAmelCase	:	Union[str, Any]="binary"  )  ->			List[Any]:
       _a    : str     					=simple_accuracy(__snake_case    ,__snake_case  )
       _a    : Dict     					=float(fa_score(y_true=__snake_case    ,y_pred=__snake_case    ,average=__snake_case  )  )
       return {
           "accuracy": acc,
           "f1": fa,
       }


def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	Optional[Any]    ,_UpperCAmelCase	:	List[str]  )  ->			Optional[Any]:
       _a    : Tuple     					={}
       for id_pred, label in zip(__snake_case    ,__snake_case  ):
              _a    : str     					=F"{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}"
              _a    : Tuple     					=id_pred['prediction']
              if question_id in question_map:
                     question_map[question_id].append((pred, label)  )
              else:
                     _a    : List[str]     					=[(pred, label)]
       _a    : List[str]     					=[], []
       for question, preds_labels in question_map.items():
              _a    : str     					=zip(*__snake_case  )
              _a    : List[Any]     					=fa_score(y_true=__snake_case    ,y_pred=__snake_case    ,average="""macro"""  )
              fas.append(__snake_case  )
              _a    : int     					=int(sum(pred == label for pred, label in preds_labels  ) == len(__snake_case  )  )
              ems.append(__snake_case  )
       _a    : int     					=float(sum(__snake_case  ) / len(__snake_case  )  )
       _a    : Optional[int]     					=sum(__snake_case  ) / len(__snake_case  )
       _a    : List[str]     					=float(fa_score(y_true=__snake_case    ,y_pred=[id_pred["""prediction"""] for id_pred in ids_preds]  )  )
       return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION ,     _KWARGS_DESCRIPTION      )
class A__       (						datasets.Metric      ):






      def __UpperCAmelCase       ( self			:Dict	)       ->					Any:




             '''simple docstring'''
             if self.config_name not in [
                 "boolq",
                 "cb",
                 "copa",
                 "multirc",
                 "record",
                 "rte",
                 "wic",
                 "wsc",
                 "wsc.fixed",
                 "axb",
                 "axg",
             ]:
                    raise KeyError(
                        """You should supply a configuration name selected in """
                        """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"""	)
             return datasets.MetricInfo(
                 description=_DESCRIPTION     ,						citation=_CITATION     ,						inputs_description=_KWARGS_DESCRIPTION     ,						features=datasets.Features(self._get_feature_types()	)     ,						codebase_urls=[]     ,						reference_urls=[]     ,						format="""numpy""" if not self.config_name == """record""" and not self.config_name == """multirc""" else None     ,						)






      def __UpperCAmelCase       ( self			:List[str]	)       ->					List[Any]:




             '''simple docstring'''
             if self.config_name == "record":
                    return {
                        "predictions": {
                            "idx": {
                                "passage": datasets.Value("""int64"""	),
                                "query": datasets.Value("""int64"""	),
                            },
                            "prediction_text": datasets.Value("""string"""	),
                        },
                        "references": {
                            "idx": {
                                "passage": datasets.Value("""int64"""	),
                                "query": datasets.Value("""int64"""	),
                            },
                            "answers": datasets.Sequence(datasets.Value("""string"""	)	),
                        },
                    }
             elif self.config_name == "multirc":
                    return {
                        "predictions": {
                            "idx": {
                                "answer": datasets.Value("""int64"""	),
                                "paragraph": datasets.Value("""int64"""	),
                                "question": datasets.Value("""int64"""	),
                            },
                            "prediction": datasets.Value("""int64"""	),
                        },
                        "references": datasets.Value("""int64"""	),
                    }
             else:
                    return {
                        "predictions": datasets.Value("""int64"""	),
                        "references": datasets.Value("""int64"""	),
                    }






      def __UpperCAmelCase       ( self			:Tuple     ,						SCREAMING_SNAKE_CASE			:Tuple     ,						SCREAMING_SNAKE_CASE			:Dict	)       ->					Optional[int]:




             '''simple docstring'''
             if self.config_name == "axb":
                    return {"matthews_correlation": matthews_corrcoef(_UpperCamelCase     ,						_UpperCamelCase	)}
             elif self.config_name == "cb":
                    return acc_and_fa(_UpperCamelCase     ,						_UpperCamelCase     ,						fa_avg="""macro"""	)
             elif self.config_name == "record":
                    _a    : str     					=[
                        {
                            'qas': [
                                {'id': ref['idx']['query'], 'answers': [{'text': ans} for ans in ref['answers']]}
                                for ref in references
                            ]
                        }
                    ]
                    _a    : Any     					={pred['idx']['query']: pred['prediction_text'] for pred in predictions}
                    return evaluate_record(_UpperCamelCase     ,						_UpperCamelCase	)[0]
             elif self.config_name == "multirc":
                    return evaluate_multirc(_UpperCamelCase     ,						_UpperCamelCase	)
             elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
                    return {"accuracy": simple_accuracy(_UpperCamelCase     ,						_UpperCamelCase	)}
             else:
                    raise KeyError(
                        """You should supply a configuration name selected in """
                        """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"""	)










import importlib
import shutil
import threading
import warnings
from typing import List

import fsspec
import fsspec.asyn

from . import compression
from .hffilesystem import HfFileSystem


__UpperCAmelCase								=						importlib.util.find_spec('s3fs') is not None

if _has_safs:
		from .safilesystem import SaFileSystem  # noqa: F401

__UpperCAmelCase								=						[
    compression.BzaFileSystem,
    compression.GzipFileSystem,
    compression.LzaFileSystem,
    compression.XzFileSystem,
    compression.ZstdFileSystem,
]

# Register custom filesystems
for fs_class in COMPRESSION_FILESYSTEMS + [HfFileSystem]:
		if fs_class.protocol in fsspec.registry and fsspec.registry[fs_class.protocol] is not fs_class:
				warnings.warn(F'A filesystem protocol was already set for {fs_class.protocol} and will be overwritten.')
		fsspec.register_implementation(fs_class.protocol, fs_class, clobber=True)




def      lowercase__      (     __snake_case    :							str  ):


					'''simple docstring'''


					if "://" in dataset_path:
										UpperCAmelCase_    :      int            =						dataset_path.split('://'  )[1]
					return dataset_path




def      lowercase__      (     __snake_case    :							fsspec.AbstractFileSystem  ):


					'''simple docstring'''


					if fs is not None and fs.protocol != "file":
										return True
					else:
										return False




def      lowercase__      (     __snake_case    :							fsspec.AbstractFileSystem     ,       __snake_case    :							str     ,       __snake_case    :							str  ):


					'''simple docstring'''


					UpperCAmelCase_    :      List[str]            =						not is_remote_filesystem(__snake_case  )
					if is_local:
										# LocalFileSystem.mv does copy + rm, it is more efficient to simply move a local directory
										shutil.move(fs._strip_protocol(__snake_case  )     ,       fs._strip_protocol(__snake_case  )  )
					else:
										fs.mv(__snake_case     ,       __snake_case     ,       recursive=__snake_case  )






def      lowercase__      (     ):


					'''simple docstring'''


					if hasattr(fsspec.asyn     ,       'reset_lock'  ):
										# for future fsspec>2022.05.0
										fsspec.asyn.reset_lock()
					else:
										UpperCAmelCase_    :      Optional[Any]            =						None
										UpperCAmelCase_    :      Union[str, Any]            =						None
										UpperCAmelCase_    :      int            =						threading.Lock()






