
from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


SCREAMING_SNAKE_CASE							:Optional[int]							 =				'\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n'

SCREAMING_SNAKE_CASE							:Union[str, Any]							 =				'\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n'

SCREAMING_SNAKE_CASE							:str							 =				'\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for \'record\': list of question-answer dictionaries with the following keys:\n            - \'idx\': index of the question as specified by the dataset\n            - \'prediction_text\': the predicted answer text\n        - for \'multirc\': list of question-answer dictionaries with the following keys:\n            - \'idx\': index of the question-answer pair as specified by the dataset\n            - \'prediction\': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for \'record\': list of question-answers dictionaries with the following keys:\n            - \'idx\': index of the question as specified by the dataset\n            - \'answers\': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for \'record\':\n        - \'exact_match\': Exact match between answer and gold answer\n        - \'f1\': F1 score\n    - for \'multirc\':\n        - \'exact_match\': Exact match between answer and gold answer\n        - \'f1_m\': Per-question macro-F1 score\n        - \'f1_a\': Average F1 score over all answers\n    - for \'axb\':\n        \'matthews_correlation\': Matthew Correlation\n    - for \'cb\':\n        - \'accuracy\': Accuracy\n        - \'f1\': F1 score\n    - for all others:\n        - \'accuracy\': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0, \'f1\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')\n    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]\n    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 1.0, \'f1\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')\n    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'matthews_correlation\': 1.0}\n'


def 						UpperCAmelCase			(	a_		,   a_						)		->	Dict:







				"""simple docstring"""



				return float((preds == labels).mean()						)


def 						UpperCAmelCase			(	a_		,   a_		,   a_="binary"						)		->	Union[str, Any]:







				"""simple docstring"""



				__A   			= simple_accuracy(a_		,   a_						)
				__A   			= float(fa_score(y_true=a_		,   y_pred=a_		,   average=a_						)						)
				return {
				    "accuracy": acc,
				    "f1": fa,
				}


def 						UpperCAmelCase			(	a_		,   a_						)		->	Union[str, Any]:







				"""simple docstring"""



				__A   			= {}
				for id_pred, label in zip(a_		,   a_						):
								__A   			= F'''{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}'''
								__A   			= id_pred["prediction"]
								if question_id in question_map:
												question_map[question_id].append((pred, label)						)
								else:
												__A   			= [(pred, label)]
				__A			,							__A   			= [], []
				for question, preds_labels in question_map.items():
								__A			,							__A   			= zip(*a_						)
								__A   			= fa_score(y_true=a_		,   y_pred=a_		,   average="macro"						)
								fas.append(a_						)
								__A   			= int(sum(pred == label for pred, label in preds_labels						) == len(a_						)						)
								ems.append(a_						)
				__A   			= float(sum(a_						) / len(a_						)						)
				__A   			= sum(a_						) / len(a_						)
				__A   			= float(fa_score(y_true=a_		,   y_pred=[id_pred["prediction"] for id_pred in ids_preds]						)						)
				return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION			,			_KWARGS_DESCRIPTION  )
class 							UpperCAmelCase      (  datasets.Metric  ):




	'''simple docstring'''





	def 				UpperCamelCase_  (       self					: List[str] ):
					if self.config_name not in [
					    "boolq",
					    "cb",
					    "copa",
					    "multirc",
					    "record",
					    "rte",
					    "wic",
					    "wsc",
					    "wsc.fixed",
					    "axb",
					    "axg",
					]:
									raise KeyError(
									    "You should supply a configuration name selected in "
									    "[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]" )
					return datasets.MetricInfo(
					    description=_DESCRIPTION							,citation=_CITATION							,inputs_description=_KWARGS_DESCRIPTION							,features=datasets.Features(self._get_feature_types() )							,codebase_urls=[]							,reference_urls=[]							,format="numpy" if not self.config_name == "record" and not self.config_name == "multirc" else None							,)






	def 				UpperCamelCase_  (       self					: List[Any] ):
					if self.config_name == "record":
									return {
									    "predictions": {
									        "idx": {
									            "passage": datasets.Value("int64" ),
									            "query": datasets.Value("int64" ),
									        },
									        "prediction_text": datasets.Value("string" ),
									    },
									    "references": {
									        "idx": {
									            "passage": datasets.Value("int64" ),
									            "query": datasets.Value("int64" ),
									        },
									        "answers": datasets.Sequence(datasets.Value("string" ) ),
									    },
									}
					elif self.config_name == "multirc":
									return {
									    "predictions": {
									        "idx": {
									            "answer": datasets.Value("int64" ),
									            "paragraph": datasets.Value("int64" ),
									            "question": datasets.Value("int64" ),
									        },
									        "prediction": datasets.Value("int64" ),
									    },
									    "references": datasets.Value("int64" ),
									}
					else:
									return {
									    "predictions": datasets.Value("int64" ),
									    "references": datasets.Value("int64" ),
									}







	def 				UpperCamelCase_  (       self					: Union[str, Any]							,A					: Union[str, Any]							,A					: Dict ):
					if self.config_name == "axb":
									return {"matthews_correlation": matthews_corrcoef(A							,A )}
					elif self.config_name == "cb":
									return acc_and_fa(A							,A							,fa_avg="macro" )
					elif self.config_name == "record":
									__A   			= [
									    {
									        "qas": [
									            {"id": ref["idx"]["query"], "answers": [{"text": ans} for ans in ref["answers"]]}
									            for ref in references
									        ]
									    }
									]
									__A   			= {pred["idx"]["query"]: pred["prediction_text"] for pred in predictions}
									return evaluate_record(A							,A )[0]
					elif self.config_name == "multirc":
									return evaluate_multirc(A							,A )
					elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
									return {"accuracy": simple_accuracy(A							,A )}
					else:
									raise KeyError(
									    "You should supply a configuration name selected in "
									    "[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]" )







import logging

from transformers.configuration_utils import PretrainedConfig


__A									=				logging.getLogger(__name__)



class    _SCREAMING_SNAKE_CASE						(	__SCREAMING_SNAKE_CASE    ):







 '''simple docstring'''
 lowercase_		=			"masked_bert"

 def __init__(self				:   Dict       , UpperCAmelCase_				:   Any=30_522       , UpperCAmelCase_				:   List[Any]=768       , UpperCAmelCase_				:   Optional[Any]=12       , UpperCAmelCase_				:   str=12       , UpperCAmelCase_				:   Tuple=3_072       , UpperCAmelCase_				:   str="gelu"       , UpperCAmelCase_				:   Any=0.1       , UpperCAmelCase_				:   Tuple=0.1       , UpperCAmelCase_				:   Optional[Any]=512       , UpperCAmelCase_				:   Union[str, Any]=2       , UpperCAmelCase_				:   str=0.02       , UpperCAmelCase_				:   str=1E-1_2       , UpperCAmelCase_				:   Union[str, Any]=0       , UpperCAmelCase_				:   str="topK"       , UpperCAmelCase_				:   List[str]="constant"       , UpperCAmelCase_				:   str=0.0       , **UpperCAmelCase_				:   int       , )       ->List[Any]:



   '''simple docstring'''



   super().__init__(pad_token_id=UpperCAmelCase_       , **UpperCAmelCase_)

   lowerCamelCase__:      Optional[int]			  =vocab_size
   lowerCamelCase__:      Dict			  =hidden_size
   lowerCamelCase__:      Optional[int]			  =num_hidden_layers
   lowerCamelCase__:      Any			  =num_attention_heads
   lowerCamelCase__:      List[Any]			  =hidden_act
   lowerCamelCase__:      str			  =intermediate_size
   lowerCamelCase__:      Dict			  =hidden_dropout_prob
   lowerCamelCase__:      str			  =attention_probs_dropout_prob
   lowerCamelCase__:      int			  =max_position_embeddings
   lowerCamelCase__:      Tuple			  =type_vocab_size
   lowerCamelCase__:      str			  =initializer_range
   lowerCamelCase__:      List[Any]			  =layer_norm_eps
   lowerCamelCase__:      str			  =pruning_method
   lowerCamelCase__:      Union[str, Any]			  =mask_init
   lowerCamelCase__:      Optional[Any]			  =mask_scale

