






import json
import os
import pickle
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers import is_faiss_available
from transformers.models.bart.configuration_bart import BartConfig
from transformers.models.bart.tokenization_bart import BartTokenizer
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.dpr.configuration_dpr import DPRConfig
from transformers.models.dpr.tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer
from transformers.models.rag.configuration_rag import RagConfig
from transformers.models.rag.retrieval_rag import CustomHFIndex, RagRetriever
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_faiss, require_sentencepiece, require_tokenizers, require_torch


if is_faiss_available():
	import faiss

@require_faiss
class     _SCREAMING_SNAKE_CASE       (			_a      ):





	def 						_A (		self				:					int	):
		UpperCamelCase     :List[Any]          =	tempfile.mkdtemp()
		UpperCamelCase     :List[str]          =	8

		# DPR tok
		UpperCamelCase     :Union[str, Any]          =	[
		    """[UNK]""",
		    """[CLS]""",
		    """[SEP]""",
		    """[PAD]""",
		    """[MASK]""",
		    """want""",
		    """##want""",
		    """##ed""",
		    """wa""",
		    """un""",
		    """runn""",
		    """##ing""",
		    """,""",
		    """low""",
		    """lowest""",
		]
		UpperCamelCase     :List[Any]          =	os.path.join(self.tmpdirname ,  """dpr_tokenizer"""	)
		os.makedirs(__lowerCamelCase ,  exist_ok=__lowerCamelCase	)
		UpperCamelCase     :List[str]          =	os.path.join(__lowerCamelCase ,  DPR_VOCAB_FILES_NAMES["""vocab_file"""]	)
		with open(self.vocab_file ,  """w""" ,  encoding="""utf-8"""	) as vocab_writer:
			vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]	)	)

		# BART tok
		UpperCamelCase     :List[str]          =	[
		    """l""",
		    """o""",
		    """w""",
		    """e""",
		    """r""",
		    """s""",
		    """t""",
		    """i""",
		    """d""",
		    """n""",
		    """\u0120""",
		    """\u0120l""",
		    """\u0120n""",
		    """\u0120lo""",
		    """\u0120low""",
		    """er""",
		    """\u0120lowest""",
		    """\u0120newer""",
		    """\u0120wider""",
		    """<unk>""",
		]
		UpperCamelCase     :Dict          =	dict(zip(__lowerCamelCase ,  range(len(__lowerCamelCase	)	)	)	)
		UpperCamelCase     :int          =	["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
		UpperCamelCase     :Union[str, Any]          =	{"""unk_token""": """<unk>"""}

		UpperCamelCase     :Optional[int]          =	os.path.join(self.tmpdirname ,  """bart_tokenizer"""	)
		os.makedirs(__lowerCamelCase ,  exist_ok=__lowerCamelCase	)
		UpperCamelCase     :List[str]          =	os.path.join(__lowerCamelCase ,  BART_VOCAB_FILES_NAMES["""vocab_file"""]	)
		UpperCamelCase     :Union[str, Any]          =	os.path.join(__lowerCamelCase ,  BART_VOCAB_FILES_NAMES["""merges_file"""]	)
		with open(self.vocab_file ,  """w""" ,  encoding="""utf-8"""	) as fp:
			fp.write(json.dumps(__lowerCamelCase	) + """\n"""	)
		with open(self.merges_file ,  """w""" ,  encoding="""utf-8"""	) as fp:
			fp.write("""\n""".join(__lowerCamelCase	)	)





	def 						_A (		self				:					List[Any]	):
		return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname ,  """dpr_tokenizer"""	)	)





	def 						_A (		self				:					Tuple	):
		return DPRContextEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname ,  """dpr_tokenizer"""	)	)





	def 						_A (		self				:					Union[str, Any]	):
		return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname ,  """bart_tokenizer"""	)	)





	def 						_A (		self				:					Tuple	):
		shutil.rmtree(self.tmpdirname	)





	def 						_A (		self				:					Tuple	):
		UpperCamelCase     :Optional[int]          =	Dataset.from_dict(
		    {
		        """id""": ["""0""", """1"""],
		        """text""": ["""foo""", """bar"""],
		        """title""": ["""Foo""", """Bar"""],
		        """embeddings""": [np.ones(self.retrieval_vector_size	), 2 * np.ones(self.retrieval_vector_size	)],
		    }	)
		dataset.add_faiss_index("""embeddings""" ,  string_factory="""Flat""" ,  metric_type=faiss.METRIC_INNER_PRODUCT	)
		return dataset





	def 						_A (		self				:					Tuple	):
		UpperCamelCase     :List[str]          =	self.get_dummy_dataset()
		UpperCamelCase     :List[str]          =	RagConfig(
		    retrieval_vector_size=self.retrieval_vector_size ,  question_encoder=DPRConfig().to_dict() ,  generator=BartConfig().to_dict() ,  )
		with patch("""transformers.models.rag.retrieval_rag.load_dataset"""	) as mock_load_dataset:
			UpperCamelCase     :List[Any]          =	dataset
			UpperCamelCase     :Dict          =	RagRetriever(
			    __lowerCamelCase ,  question_encoder_tokenizer=self.get_dpr_tokenizer() ,  generator_tokenizer=self.get_bart_tokenizer() ,  )
		return retriever





	def 						_A (		self				:					Optional[Any] ,  __lowerCamelCase				:					bool	):
		UpperCamelCase     :Union[str, Any]          =	self.get_dummy_dataset()
		UpperCamelCase     :int          =	RagConfig(
		    retrieval_vector_size=self.retrieval_vector_size ,  question_encoder=DPRConfig().to_dict() ,  generator=BartConfig().to_dict() ,  index_name="""custom""" ,  )
		if from_disk:
			UpperCamelCase     :Any          =	os.path.join(self.tmpdirname ,  """dataset"""	)
			UpperCamelCase     :List[str]          =	os.path.join(self.tmpdirname ,  """index.faiss"""	)
			dataset.get_index("""embeddings"""	).save(os.path.join(self.tmpdirname ,  """index.faiss"""	)	)
			dataset.drop_index("""embeddings"""	)
			dataset.save_to_disk(os.path.join(self.tmpdirname ,  """dataset"""	)	)
			del dataset
			UpperCamelCase     :Optional[Any]          =	RagRetriever(
			    __lowerCamelCase ,  question_encoder_tokenizer=self.get_dpr_tokenizer() ,  generator_tokenizer=self.get_bart_tokenizer() ,  )
		else:
			UpperCamelCase     :Any          =	RagRetriever(
			    __lowerCamelCase ,  question_encoder_tokenizer=self.get_dpr_tokenizer() ,  generator_tokenizer=self.get_bart_tokenizer() ,  index=CustomHFIndex(config.retrieval_vector_size ,  __lowerCamelCase	) ,  )
		return retriever





	def 						_A (		self				:					Any	):
		UpperCamelCase     :List[Any]          =	Dataset.from_dict(
		    {
		        """id""": ["""0""", """1"""],
		        """text""": ["""foo""", """bar"""],
		        """title""": ["""Foo""", """Bar"""],
		        """embeddings""": [np.ones(self.retrieval_vector_size + 1	), 2 * np.ones(self.retrieval_vector_size + 1	)],
		    }	)
		dataset.add_faiss_index("""embeddings""" ,  string_factory="""Flat""" ,  metric_type=faiss.METRIC_INNER_PRODUCT	)

		UpperCamelCase     :Optional[Any]          =	os.path.join(self.tmpdirname ,  """hf_bert_base.hnswSQ8_correct_phi_128.c_index"""	)
		dataset.save_faiss_index("""embeddings""" ,  index_file_name + """.index.dpr"""	)
		pickle.dump(dataset["""id"""] ,  open(index_file_name + """.index_meta.dpr""" ,  """wb"""	)	)

		UpperCamelCase     :Union[str, Any]          =	os.path.join(self.tmpdirname ,  """psgs_w100.tsv.pkl"""	)
		UpperCamelCase     :Optional[int]          =	{sample["""id"""]: [sample["""text"""], sample["""title"""]] for sample in dataset}
		pickle.dump(__lowerCamelCase ,  open(__lowerCamelCase ,  """wb"""	)	)

		UpperCamelCase     :Dict          =	RagConfig(
		    retrieval_vector_size=self.retrieval_vector_size ,  question_encoder=DPRConfig().to_dict() ,  generator=BartConfig().to_dict() ,  index_name="""legacy""" ,  index_path=self.tmpdirname ,  )
		UpperCamelCase     :List[Any]          =	RagRetriever(
		    __lowerCamelCase ,  question_encoder_tokenizer=self.get_dpr_tokenizer() ,  generator_tokenizer=self.get_bart_tokenizer()	)
		return retriever





	def 						_A (		self				:					Dict	):
		UpperCamelCase     :Union[str, Any]          =	1
		UpperCamelCase     :Dict          =	self.get_dummy_canonical_hf_index_retriever()
		UpperCamelCase     :Union[str, Any]          =	np.array(
		    [np.ones(self.retrieval_vector_size	), -np.ones(self.retrieval_vector_size	)] ,  dtype=np.floataa	)
		UpperCamelCase     ,      UpperCamelCase     ,      UpperCamelCase     :List[str]          =	retriever.retrieve(__lowerCamelCase ,  n_docs=__lowerCamelCase	)
		self.assertEqual(retrieved_doc_embeds.shape ,  (2, n_docs, self.retrieval_vector_size)	)
		self.assertEqual(len(__lowerCamelCase	) ,  2	)
		self.assertEqual(sorted(doc_dicts[0]	) ,  ["""embeddings""", """id""", """text""", """title"""]	)
		self.assertEqual(len(doc_dicts[0]["""id"""]	) ,  __lowerCamelCase	)
		self.assertEqual(doc_dicts[0]["""id"""][0] ,  """1"""	)  # max inner product is reached with second doc
		self.assertEqual(doc_dicts[1]["""id"""][0] ,  """0"""	)  # max inner product is reached with first doc
		self.assertListEqual(doc_ids.tolist() ,  [[1], [0]]	)





	def 						_A (		self				:					Optional[Any]	):
		UpperCamelCase     :List[Any]          =	self.get_dummy_canonical_hf_index_retriever()
		with tempfile.TemporaryDirectory() as tmp_dirname:
			with patch("""transformers.models.rag.retrieval_rag.load_dataset"""	) as mock_load_dataset:
				UpperCamelCase     :List[str]          =	self.get_dummy_dataset()
				retriever.save_pretrained(__lowerCamelCase	)
				UpperCamelCase     :int          =	RagRetriever.from_pretrained(__lowerCamelCase	)
				self.assertIsInstance(__lowerCamelCase ,  __lowerCamelCase	)
				UpperCamelCase     :List[Any]          =	np.array(
				    [np.ones(self.retrieval_vector_size	), -np.ones(self.retrieval_vector_size	)] ,  dtype=np.floataa	)
				UpperCamelCase     :Dict          =	retriever.retrieve(__lowerCamelCase ,  n_docs=1	)
				self.assertTrue(out is not None	)





	def 						_A (		self				:					Tuple	):
		UpperCamelCase     :int          =	1
		UpperCamelCase     :Tuple          =	self.get_dummy_custom_hf_index_retriever(from_disk=__lowerCamelCase	)
		UpperCamelCase     :Union[str, Any]          =	np.array(
		    [np.ones(self.retrieval_vector_size	), -np.ones(self.retrieval_vector_size	)] ,  dtype=np.floataa	)
		UpperCamelCase     ,      UpperCamelCase     ,      UpperCamelCase     :Union[str, Any]          =	retriever.retrieve(__lowerCamelCase ,  n_docs=__lowerCamelCase	)
		self.assertEqual(retrieved_doc_embeds.shape ,  (2, n_docs, self.retrieval_vector_size)	)
		self.assertEqual(len(__lowerCamelCase	) ,  2	)
		self.assertEqual(sorted(doc_dicts[0]	) ,  ["""embeddings""", """id""", """text""", """title"""]	)
		self.assertEqual(len(doc_dicts[0]["""id"""]	) ,  __lowerCamelCase	)
		self.assertEqual(doc_dicts[0]["""id"""][0] ,  """1"""	)  # max inner product is reached with second doc
		self.assertEqual(doc_dicts[1]["""id"""][0] ,  """0"""	)  # max inner product is reached with first doc
		self.assertListEqual(doc_ids.tolist() ,  [[1], [0]]	)





	def 						_A (		self				:					Optional[int]	):
		UpperCamelCase     :Union[str, Any]          =	self.get_dummy_custom_hf_index_retriever(from_disk=__lowerCamelCase	)
		with tempfile.TemporaryDirectory() as tmp_dirname:
			retriever.save_pretrained(__lowerCamelCase	)
			UpperCamelCase     :str          =	RagRetriever.from_pretrained(__lowerCamelCase	)
			self.assertIsInstance(__lowerCamelCase ,  __lowerCamelCase	)
			UpperCamelCase     :List[str]          =	np.array(
			    [np.ones(self.retrieval_vector_size	), -np.ones(self.retrieval_vector_size	)] ,  dtype=np.floataa	)
			UpperCamelCase     :Optional[int]          =	retriever.retrieve(__lowerCamelCase ,  n_docs=1	)
			self.assertTrue(out is not None	)





	def 						_A (		self				:					Any	):
		UpperCamelCase     :List[str]          =	1
		UpperCamelCase     :Dict          =	self.get_dummy_custom_hf_index_retriever(from_disk=__lowerCamelCase	)
		UpperCamelCase     :Union[str, Any]          =	np.array(
		    [np.ones(self.retrieval_vector_size	), -np.ones(self.retrieval_vector_size	)] ,  dtype=np.floataa	)
		UpperCamelCase     ,      UpperCamelCase     ,      UpperCamelCase     :str          =	retriever.retrieve(__lowerCamelCase ,  n_docs=__lowerCamelCase	)
		self.assertEqual(retrieved_doc_embeds.shape ,  (2, n_docs, self.retrieval_vector_size)	)
		self.assertEqual(len(__lowerCamelCase	) ,  2	)
		self.assertEqual(sorted(doc_dicts[0]	) ,  ["""embeddings""", """id""", """text""", """title"""]	)
		self.assertEqual(len(doc_dicts[0]["""id"""]	) ,  __lowerCamelCase	)
		self.assertEqual(doc_dicts[0]["""id"""][0] ,  """1"""	)  # max inner product is reached with second doc
		self.assertEqual(doc_dicts[1]["""id"""][0] ,  """0"""	)  # max inner product is reached with first doc
		self.assertListEqual(doc_ids.tolist() ,  [[1], [0]]	)





	def 						_A (		self				:					List[Any]	):
		UpperCamelCase     :List[Any]          =	self.get_dummy_custom_hf_index_retriever(from_disk=__lowerCamelCase	)
		with tempfile.TemporaryDirectory() as tmp_dirname:
			retriever.save_pretrained(__lowerCamelCase	)
			UpperCamelCase     :Optional[int]          =	RagRetriever.from_pretrained(__lowerCamelCase	)
			self.assertIsInstance(__lowerCamelCase ,  __lowerCamelCase	)
			UpperCamelCase     :Tuple          =	np.array(
			    [np.ones(self.retrieval_vector_size	), -np.ones(self.retrieval_vector_size	)] ,  dtype=np.floataa	)
			UpperCamelCase     :Any          =	retriever.retrieve(__lowerCamelCase ,  n_docs=1	)
			self.assertTrue(out is not None	)





	def 						_A (		self				:					Optional[Any]	):
		UpperCamelCase     :Any          =	1
		UpperCamelCase     :List[Any]          =	self.get_dummy_legacy_index_retriever()
		UpperCamelCase     :Dict          =	np.array(
		    [np.ones(self.retrieval_vector_size	), -np.ones(self.retrieval_vector_size	)] ,  dtype=np.floataa	)
		UpperCamelCase     ,      UpperCamelCase     ,      UpperCamelCase     :Any          =	retriever.retrieve(__lowerCamelCase ,  n_docs=__lowerCamelCase	)
		self.assertEqual(retrieved_doc_embeds.shape ,  (2, n_docs, self.retrieval_vector_size)	)
		self.assertEqual(len(__lowerCamelCase	) ,  2	)
		self.assertEqual(sorted(doc_dicts[0]	) ,  ["""text""", """title"""]	)
		self.assertEqual(len(doc_dicts[0]["""text"""]	) ,  __lowerCamelCase	)
		self.assertEqual(doc_dicts[0]["""text"""][0] ,  """bar"""	)  # max inner product is reached with second doc
		self.assertEqual(doc_dicts[1]["""text"""][0] ,  """foo"""	)  # max inner product is reached with first doc
		self.assertListEqual(doc_ids.tolist() ,  [[1], [0]]	)





	def 						_A (		self				:					Optional[int]	):
		UpperCamelCase     :Optional[int]          =	self.get_dummy_legacy_index_retriever()
		with tempfile.TemporaryDirectory() as tmp_dirname:
			retriever.save_pretrained(__lowerCamelCase	)
			UpperCamelCase     :List[Any]          =	RagRetriever.from_pretrained(__lowerCamelCase	)
			self.assertIsInstance(__lowerCamelCase ,  __lowerCamelCase	)
			UpperCamelCase     :Optional[Any]          =	np.array(
			    [np.ones(self.retrieval_vector_size	), -np.ones(self.retrieval_vector_size	)] ,  dtype=np.floataa	)
			UpperCamelCase     :Tuple          =	retriever.retrieve(__lowerCamelCase ,  n_docs=1	)
			self.assertTrue(out is not None	)





	@require_torch
	@require_tokenizers
	@require_sentencepiece
	def 						_A (		self				:					Any	):
		import torch

		UpperCamelCase     :List[Any]          =	1
		UpperCamelCase     :List[Any]          =	self.get_dummy_canonical_hf_index_retriever()
		UpperCamelCase     :int          =	[[5, 7], [10, 11]]
		UpperCamelCase     :Dict          =	np.array(
		    [np.ones(self.retrieval_vector_size	), -np.ones(self.retrieval_vector_size	)] ,  dtype=np.floataa	)
		UpperCamelCase     :Optional[int]          =	retriever(__lowerCamelCase ,  __lowerCamelCase ,  prefix=retriever.config.generator.prefix ,  n_docs=__lowerCamelCase	)
		UpperCamelCase     ,      UpperCamelCase     ,      UpperCamelCase     :Optional[int]          =	(
		    out["""context_input_ids"""],
		    out["""context_attention_mask"""],
		    out["""retrieved_doc_embeds"""],
		)
		self.assertEqual(retrieved_doc_embeds.shape ,  (2, n_docs, self.retrieval_vector_size)	)
		self.assertIsInstance(__lowerCamelCase ,  __lowerCamelCase	)
		self.assertIsInstance(__lowerCamelCase ,  __lowerCamelCase	)
		self.assertIsInstance(__lowerCamelCase ,  np.ndarray	)

		UpperCamelCase     :List[str]          =	retriever(
		    __lowerCamelCase ,  __lowerCamelCase ,  prefix=retriever.config.generator.prefix ,  n_docs=__lowerCamelCase ,  return_tensors="""pt""" ,  )
		UpperCamelCase     ,      UpperCamelCase     ,      UpperCamelCase     ,      UpperCamelCase     :Optional[int]          =	(  # noqa: F841
		    out["""context_input_ids"""],
		    out["""context_attention_mask"""],
		    out["""retrieved_doc_embeds"""],
		    out["""doc_ids"""],
		)
		self.assertEqual(retrieved_doc_embeds.shape ,  (2, n_docs, self.retrieval_vector_size)	)
		self.assertIsInstance(__lowerCamelCase ,  torch.Tensor	)
		self.assertIsInstance(__lowerCamelCase ,  torch.Tensor	)
		self.assertIsInstance(__lowerCamelCase ,  torch.Tensor	)





	@require_torch
	@require_tokenizers
	@require_sentencepiece
	def 						_A (		self				:					List[str]	):
		UpperCamelCase     :Dict          =	self.get_dpr_ctx_encoder_tokenizer()
		UpperCamelCase     :Dict          =	1
		UpperCamelCase     :List[Any]          =	self.get_dummy_custom_hf_index_retriever(from_disk=__lowerCamelCase	)
		retriever.set_ctx_encoder_tokenizer(__lowerCamelCase	)

		UpperCamelCase     :int          =	[[5, 7], [10, 11]]
		UpperCamelCase     :List[Any]          =	np.array(
		    [np.ones(self.retrieval_vector_size	), -np.ones(self.retrieval_vector_size	)] ,  dtype=np.floataa	)
		UpperCamelCase     :Optional[Any]          =	retriever(__lowerCamelCase ,  __lowerCamelCase ,  prefix=retriever.config.generator.prefix ,  n_docs=__lowerCamelCase	)

		self.assertEqual(
		    len(__lowerCamelCase	) ,  6	)  # check whether the retriever output consist of 6 attributes including tokenized docs
		self.assertEqual(
		    all(k in out for k in ("""tokenized_doc_ids""", """tokenized_doc_attention_mask""")	) ,  __lowerCamelCase	)  # check for doc token related keys in dictionary.








from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


UpperCAmelCase_ :						int      =      '''\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
'''

UpperCAmelCase_ :						Optional[Any]      =      '''\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
'''

UpperCAmelCase_ :						int      =      '''
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for \'record\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'prediction_text\': the predicted answer text
        - for \'multirc\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question-answer pair as specified by the dataset
            - \'prediction\': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for \'record\': list of question-answers dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'answers\': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for \'record\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1\': F1 score
    - for \'multirc\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1_m\': Per-question macro-F1 score
        - \'f1_a\': Average F1 score over all answers
    - for \'axb\':
        \'matthews_correlation\': Matthew Correlation
    - for \'cb\':
        - \'accuracy\': Accuracy
        - \'f1\': F1 score
    - for all others:
        - \'accuracy\': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')
    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]
    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')
    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'matthews_correlation\': 1.0}
'''

def    SCREAMING_SNAKE_CASE_							(							__magic_name__					:  Optional[Any]	,			__magic_name__					:  Optional[Any]					)		->			Union[str, Any]:






	"""simple docstring"""



	return float((preds == labels).mean()					)

def    SCREAMING_SNAKE_CASE_							(							__magic_name__					:  Dict	,			__magic_name__					:  int	,			__magic_name__					:  Any="binary"					)		->			Dict:






	"""simple docstring"""



	UpperCamelCase     :List[str]          =	simple_accuracy(__magic_name__	,			__magic_name__					)
	UpperCamelCase     :Dict          =	float(fa_score(y_true=__magic_name__	,			y_pred=__magic_name__	,			average=__magic_name__					)					)
	return {
	    "accuracy": acc,
	    "f1": fa,
	}

def    SCREAMING_SNAKE_CASE_							(							__magic_name__					:  List[Any]	,			__magic_name__					:  Optional[Any]					)		->			Optional[Any]:






	"""simple docstring"""



	UpperCamelCase     :Optional[Any]          =	{}
	for id_pred, label in zip(__magic_name__	,			__magic_name__					):
		UpperCamelCase     :str          =	f"""{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}"""
		UpperCamelCase     :Union[str, Any]          =	id_pred["""prediction"""]
		if question_id in question_map:
			question_map[question_id].append((pred, label)					)
		else:
			UpperCamelCase     :Dict          =	[(pred, label)]
	UpperCamelCase     ,      UpperCamelCase     :Optional[int]          =	[], []
	for question, preds_labels in question_map.items():
		UpperCamelCase     ,      UpperCamelCase     :Optional[Any]          =	zip(*__magic_name__					)
		UpperCamelCase     :Optional[int]          =	fa_score(y_true=__magic_name__	,			y_pred=__magic_name__	,			average="""macro"""					)
		fas.append(__magic_name__					)
		UpperCamelCase     :int          =	int(sum(pred == label for pred, label in preds_labels					) == len(__magic_name__					)					)
		ems.append(__magic_name__					)
	UpperCamelCase     :Optional[int]          =	float(sum(__magic_name__					) / len(__magic_name__					)					)
	UpperCamelCase     :str          =	sum(__magic_name__					) / len(__magic_name__					)
	UpperCamelCase     :Tuple          =	float(fa_score(y_true=__magic_name__	,			y_pred=[id_pred["""prediction"""] for id_pred in ids_preds]					)					)
	return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION		,   _KWARGS_DESCRIPTION      )
class     _SCREAMING_SNAKE_CASE       (			datasets.Metric      ):





	def 						_A (		self				:					str	):
		if self.config_name not in [
		    "boolq",
		    "cb",
		    "copa",
		    "multirc",
		    "record",
		    "rte",
		    "wic",
		    "wsc",
		    "wsc.fixed",
		    "axb",
		    "axg",
		]:
			raise KeyError(
			    """You should supply a configuration name selected in """
			    """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"""	)
		return datasets.MetricInfo(
		    description=_DESCRIPTION ,  citation=_CITATION ,  inputs_description=_KWARGS_DESCRIPTION ,  features=datasets.Features(self._get_feature_types()	) ,  codebase_urls=[] ,  reference_urls=[] ,  format="""numpy""" if not self.config_name == """record""" and not self.config_name == """multirc""" else None ,  )





	def 						_A (		self				:					Optional[Any]	):
		if self.config_name == "record":
			return {
			    "predictions": {
			        "idx": {
			            "passage": datasets.Value("""int64"""	),
			            "query": datasets.Value("""int64"""	),
			        },
			        "prediction_text": datasets.Value("""string"""	),
			    },
			    "references": {
			        "idx": {
			            "passage": datasets.Value("""int64"""	),
			            "query": datasets.Value("""int64"""	),
			        },
			        "answers": datasets.Sequence(datasets.Value("""string"""	)	),
			    },
			}
		elif self.config_name == "multirc":
			return {
			    "predictions": {
			        "idx": {
			            "answer": datasets.Value("""int64"""	),
			            "paragraph": datasets.Value("""int64"""	),
			            "question": datasets.Value("""int64"""	),
			        },
			        "prediction": datasets.Value("""int64"""	),
			    },
			    "references": datasets.Value("""int64"""	),
			}
		else:
			return {
			    "predictions": datasets.Value("""int64"""	),
			    "references": datasets.Value("""int64"""	),
			}





	def 						_A (		self				:					List[str] ,  __lowerCamelCase				:					Tuple ,  __lowerCamelCase				:					str	):
		if self.config_name == "axb":
			return {"matthews_correlation": matthews_corrcoef(__lowerCamelCase ,  __lowerCamelCase	)}
		elif self.config_name == "cb":
			return acc_and_fa(__lowerCamelCase ,  __lowerCamelCase ,  fa_avg="""macro"""	)
		elif self.config_name == "record":
			UpperCamelCase     :Optional[Any]          =	[
			    {
			        """qas""": [
			            {"""id""": ref["""idx"""]["""query"""], """answers""": [{"""text""": ans} for ans in ref["""answers"""]]}
			            for ref in references
			        ]
			    }
			]
			UpperCamelCase     :Tuple          =	{pred["""idx"""]["""query"""]: pred["""prediction_text"""] for pred in predictions}
			return evaluate_record(__lowerCamelCase ,  __lowerCamelCase	)[0]
		elif self.config_name == "multirc":
			return evaluate_multirc(__lowerCamelCase ,  __lowerCamelCase	)
		elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
			return {"accuracy": simple_accuracy(__lowerCamelCase ,  __lowerCamelCase	)}
		else:
			raise KeyError(
			    """You should supply a configuration name selected in """
			    """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"""	)
