






from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


__UpperCAmelCase								=						'\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n'

__UpperCAmelCase								=						'\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n'

__UpperCAmelCase								=						'\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for \'record\': list of question-answer dictionaries with the following keys:\n            - \'idx\': index of the question as specified by the dataset\n            - \'prediction_text\': the predicted answer text\n        - for \'multirc\': list of question-answer dictionaries with the following keys:\n            - \'idx\': index of the question-answer pair as specified by the dataset\n            - \'prediction\': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for \'record\': list of question-answers dictionaries with the following keys:\n            - \'idx\': index of the question as specified by the dataset\n            - \'answers\': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for \'record\':\n        - \'exact_match\': Exact match between answer and gold answer\n        - \'f1\': F1 score\n    - for \'multirc\':\n        - \'exact_match\': Exact match between answer and gold answer\n        - \'f1_m\': Per-question macro-F1 score\n        - \'f1_a\': Average F1 score over all answers\n    - for \'axb\':\n        \'matthews_correlation\': Matthew Correlation\n    - for \'cb\':\n        - \'accuracy\': Accuracy\n        - \'f1\': F1 score\n    - for all others:\n        - \'accuracy\': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0, \'f1\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')\n    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]\n    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 1.0, \'f1\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')\n    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'matthews_correlation\': 1.0}\n'




def      lowercase__      (     __snake_case    :							int     ,       __snake_case    :							Dict  ):


					'''simple docstring'''


					return float((preds == labels).mean()  )




def      lowercase__      (     __snake_case    :							Tuple     ,       __snake_case    :							Union[str, Any]     ,       __snake_case    :							Union[str, Any]="binary"  ):


					'''simple docstring'''


					UpperCAmelCase_    :      str            =						simple_accuracy(__snake_case     ,       __snake_case  )
					UpperCAmelCase_    :      Dict            =						float(fa_score(y_true=__snake_case     ,       y_pred=__snake_case     ,       average=__snake_case  )  )
					return {
					    "accuracy": acc,
					    "f1": fa,
					}




def      lowercase__      (     __snake_case    :							Optional[Any]     ,       __snake_case    :							List[str]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Tuple            =						{}
					for id_pred, label in zip(__snake_case     ,       __snake_case  ):
										UpperCAmelCase_    :      str            =						F"{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}"
										UpperCAmelCase_    :      Tuple            =						id_pred['prediction']
										if question_id in question_map:
															question_map[question_id].append((pred, label)  )
										else:
															UpperCAmelCase_    :      List[str]            =						[(pred, label)]
					UpperCAmelCase_       ,  UpperCAmelCase_    :      List[str]            =						[], []
					for question, preds_labels in question_map.items():
										UpperCAmelCase_       ,  UpperCAmelCase_    :      str            =						zip(*__snake_case  )
										UpperCAmelCase_    :      List[Any]            =						fa_score(y_true=__snake_case     ,       y_pred=__snake_case     ,       average='macro'  )
										fas.append(__snake_case  )
										UpperCAmelCase_    :      int            =						int(sum(pred == label for pred, label in preds_labels  ) == len(__snake_case  )  )
										ems.append(__snake_case  )
					UpperCAmelCase_    :      int            =						float(sum(__snake_case  ) / len(__snake_case  )  )
					UpperCAmelCase_    :      Optional[int]            =						sum(__snake_case  ) / len(__snake_case  )
					UpperCAmelCase_    :      List[str]            =						float(fa_score(y_true=__snake_case     ,       y_pred=[id_pred['prediction'] for id_pred in ids_preds]  )  )
					return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION ,  _KWARGS_DESCRIPTION			)
class   lowerCamelCase      (datasets.Metric			):







		'''simple docstring'''



		def 					__UpperCAmelCase			(							self   )	->							Any:
							if self.config_name not in [
							    "boolq",
							    "cb",
							    "copa",
							    "multirc",
							    "record",
							    "rte",
							    "wic",
							    "wsc",
							    "wsc.fixed",
							    "axb",
							    "axg",
							]:
												raise KeyError(
												    'You should supply a configuration name selected in '
												    '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'   )
							return datasets.MetricInfo(
							    description=_DESCRIPTION				,		citation=_CITATION				,		inputs_description=_KWARGS_DESCRIPTION				,		features=datasets.Features(self._get_feature_types()   )				,		codebase_urls=[]				,		reference_urls=[]				,		format='numpy' if not self.config_name == 'record' and not self.config_name == 'multirc' else None				,		)



		def 					__UpperCAmelCase			(							self   )	->							List[Any]:
							if self.config_name == "record":
												return {
												    "predictions": {
												        "idx": {
												            "passage": datasets.Value('int64'   ),
												            "query": datasets.Value('int64'   ),
												        },
												        "prediction_text": datasets.Value('string'   ),
												    },
												    "references": {
												        "idx": {
												            "passage": datasets.Value('int64'   ),
												            "query": datasets.Value('int64'   ),
												        },
												        "answers": datasets.Sequence(datasets.Value('string'   )   ),
												    },
												}
							elif self.config_name == "multirc":
												return {
												    "predictions": {
												        "idx": {
												            "answer": datasets.Value('int64'   ),
												            "paragraph": datasets.Value('int64'   ),
												            "question": datasets.Value('int64'   ),
												        },
												        "prediction": datasets.Value('int64'   ),
												    },
												    "references": datasets.Value('int64'   ),
												}
							else:
												return {
												    "predictions": datasets.Value('int64'   ),
												    "references": datasets.Value('int64'   ),
												}



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase   )	->							Optional[int]:
							if self.config_name == "axb":
												return {"matthews_correlation": matthews_corrcoef(_UpperCamelCase				,		_UpperCamelCase   )}
							elif self.config_name == "cb":
												return acc_and_fa(_UpperCamelCase				,		_UpperCamelCase				,		fa_avg='macro'   )
							elif self.config_name == "record":
												UpperCAmelCase_    :      str            =						[
												    {
												        'qas': [
												            {'id': ref['idx']['query'], 'answers': [{'text': ans} for ans in ref['answers']]}
												            for ref in references
												        ]
												    }
												]
												UpperCAmelCase_    :      Any            =						{pred['idx']['query']: pred['prediction_text'] for pred in predictions}
												return evaluate_record(_UpperCamelCase				,		_UpperCamelCase   )[0]
							elif self.config_name == "multirc":
												return evaluate_multirc(_UpperCamelCase				,		_UpperCamelCase   )
							elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
												return {"accuracy": simple_accuracy(_UpperCamelCase				,		_UpperCamelCase   )}
							else:
												raise KeyError(
												    'You should supply a configuration name selected in '
												    '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'   )














from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__UpperCAmelCase								=						{'configuration_vit_msn': ['VIT_MSN_PRETRAINED_CONFIG_ARCHIVE_MAP', 'ViTMSNConfig']}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__UpperCAmelCase								=						[
		    'VIT_MSN_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'ViTMSNModel',
		    'ViTMSNForImageClassification',
		    'ViTMSNPreTrainedModel',
		]

if TYPE_CHECKING:
		from .configuration_vit_msn import VIT_MSN_PRETRAINED_CONFIG_ARCHIVE_MAP, ViTMSNConfig

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_vit_msn import (
				    VIT_MSN_PRETRAINED_MODEL_ARCHIVE_LIST,
				    ViTMSNForImageClassification,
				    ViTMSNModel,
				    ViTMSNPreTrainedModel,
				)

else:
		import sys

		__UpperCAmelCase								=						_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)






