"""simple docstring"""






import json
import os
import unittest

from transformers import BatchEncoding, MvpTokenizer, MvpTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin, filter_roberta_detectors
@require_tokenizers
class 				snake_case       ( __UpperCAmelCase      ,    unittest.TestCase				):




					"""simple docstring"""

					snake_case__				    = MvpTokenizer
					snake_case__				    = MvpTokenizerFast
					snake_case__				    = True
					snake_case__				    = filter_roberta_detectors







					def 		__lowerCAmelCase						(				self   :	Dict			):
									super().setUp()
									UpperCAmelCase__             =							[
									    'l',
									    'o',
									    'w',
									    'e',
									    'r',
									    's',
									    't',
									    'i',
									    'd',
									    'n',
									    '\u0120',
									    '\u0120l',
									    '\u0120n',
									    '\u0120lo',
									    '\u0120low',
									    'er',
									    '\u0120lowest',
									    '\u0120newer',
									    '\u0120wider',
									    '<unk>',
									]
									UpperCAmelCase__             =							dict(zip(lowerCamelCase__    ,range(len(lowerCamelCase__			)			)			)			)
									UpperCAmelCase__             =							['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
									UpperCAmelCase__             =							{'unk_token': '<unk>'}

									UpperCAmelCase__             =							os.path.join(self.tmpdirname    ,VOCAB_FILES_NAMES['vocab_file']			)
									UpperCAmelCase__             =							os.path.join(self.tmpdirname    ,VOCAB_FILES_NAMES['merges_file']			)
									with open(self.vocab_file    ,'w'    ,encoding='utf-8'			) as fp:
													fp.write(json.dumps(lowerCamelCase__			) + '\n'			)
									with open(self.merges_file    ,'w'    ,encoding='utf-8'			) as fp:
													fp.write('\n'.join(lowerCamelCase__			)			)







					def 		__lowerCAmelCase						(				self   :	Union[str, Any]    ,**lowerCamelCase__   :	List[Any]			):
									kwargs.update(self.special_tokens_map			)
									return self.tokenizer_class.from_pretrained(self.tmpdirname    ,**lowerCamelCase__			)







					def 		__lowerCAmelCase						(				self   :	str    ,**lowerCamelCase__   :	Optional[Any]			):
									kwargs.update(self.special_tokens_map			)
									return self.rust_tokenizer_class.from_pretrained(self.tmpdirname    ,**lowerCamelCase__			)







					def 		__lowerCAmelCase						(				self   :	Dict    ,lowerCamelCase__   :	List[Any]			):
									return "lower newer", "lower newer"







					@cached_property
					def 		__lowerCAmelCase						(				self   :	Optional[int]			):
									return MvpTokenizer.from_pretrained('RUCAIBox/mvp'			)







					@cached_property
					def 		__lowerCAmelCase						(				self   :	Dict			):
									return MvpTokenizerFast.from_pretrained('RUCAIBox/mvp'			)







					@require_torch
					def 		__lowerCAmelCase						(				self   :	Union[str, Any]			):
									UpperCAmelCase__             =							['A long paragraph for summarization.', 'Another paragraph for summarization.']
									UpperCAmelCase__             =							[0, 250, 251, 17_818, 13, 39_186, 1_938, 4, 2]

									for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
													UpperCAmelCase__             =							tokenizer(lowerCamelCase__    ,max_length=len(lowerCamelCase__			)    ,padding=lowerCamelCase__    ,return_tensors='pt'			)
													self.assertIsInstance(lowerCamelCase__    ,lowerCamelCase__			)

													self.assertEqual((2, 9)    ,batch.input_ids.shape			)
													self.assertEqual((2, 9)    ,batch.attention_mask.shape			)
													UpperCAmelCase__             =							batch.input_ids.tolist()[0]
													self.assertListEqual(lowerCamelCase__    ,lowerCamelCase__			)
													# Test that special tokens are reset







					@require_torch
					def 		__lowerCAmelCase						(				self   :	Tuple			):
									UpperCAmelCase__             =							['A long paragraph for summarization.', 'Another paragraph for summarization.']
									for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
													UpperCAmelCase__             =							tokenizer(lowerCamelCase__    ,padding=lowerCamelCase__    ,return_tensors='pt'			)
													# check if input_ids are returned and no labels
													self.assertIn('input_ids'    ,lowerCamelCase__			)
													self.assertIn('attention_mask'    ,lowerCamelCase__			)
													self.assertNotIn('labels'    ,lowerCamelCase__			)
													self.assertNotIn('decoder_attention_mask'    ,lowerCamelCase__			)







					@require_torch
					def 		__lowerCAmelCase						(				self   :	Union[str, Any]			):
									UpperCAmelCase__             =							[
									    'Summary of the text.',
									    'Another summary.',
									]
									for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
													UpperCAmelCase__             =							tokenizer(text_target=lowerCamelCase__    ,max_length=32    ,padding='max_length'    ,return_tensors='pt'			)
													self.assertEqual(32    ,targets['input_ids'].shape[1]			)







					@require_torch
					def 		__lowerCAmelCase						(				self   :	List[str]			):
									for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
													UpperCAmelCase__             =							tokenizer(
													    ['I am a small frog' * 1_024, 'I am a small frog']    ,padding=lowerCamelCase__    ,truncation=lowerCamelCase__    ,return_tensors='pt'			)
													self.assertIsInstance(lowerCamelCase__    ,lowerCamelCase__			)
													self.assertEqual(batch.input_ids.shape    ,(2, 1_024)			)







					@require_torch
					def 		__lowerCAmelCase						(				self   :	Dict			):
									UpperCAmelCase__             =							['A long paragraph for summarization.']
									UpperCAmelCase__             =							[
									    'Summary of the text.',
									]
									for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
													UpperCAmelCase__             =							tokenizer(lowerCamelCase__    ,text_target=lowerCamelCase__    ,return_tensors='pt'			)
													UpperCAmelCase__             =							inputs['input_ids']
													UpperCAmelCase__             =							inputs['labels']
													self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()			)
													self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()			)
													self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()			)
													self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()			)







					def 		__lowerCAmelCase						(				self   :	Any			):
									pass







					def 		__lowerCAmelCase						(				self   :	Tuple			):
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
																	UpperCAmelCase__             =							self.rust_tokenizer_class.from_pretrained(lowerCamelCase__    ,**lowerCamelCase__			)
																	UpperCAmelCase__             =							self.tokenizer_class.from_pretrained(lowerCamelCase__    ,**lowerCamelCase__			)
																	UpperCAmelCase__             =							'A, <mask> AllenNLP sentence.'
																	UpperCAmelCase__             =							tokenizer_r.encode_plus(lowerCamelCase__    ,add_special_tokens=lowerCamelCase__    ,return_token_type_ids=lowerCamelCase__			)
																	UpperCAmelCase__             =							tokenizer_p.encode_plus(lowerCamelCase__    ,add_special_tokens=lowerCamelCase__    ,return_token_type_ids=lowerCamelCase__			)

																	# token_type_ids should put 0 everywhere
																	self.assertEqual(sum(tokens_r['token_type_ids']			)    ,sum(tokens_p['token_type_ids']			)			)

																	# attention_mask should put 1 everywhere, so sum over length should be 1
																	self.assertEqual(
																	    sum(tokens_r['attention_mask']			) / len(tokens_r['attention_mask']			)    ,sum(tokens_p['attention_mask']			) / len(tokens_p['attention_mask']			)    ,)

																	UpperCAmelCase__             =							tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids']			)
																	UpperCAmelCase__             =							tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids']			)

																	# Rust correctly handles the space before the mask while python doesnt
																	self.assertSequenceEqual(tokens_p['input_ids']    ,[0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]			)
																	self.assertSequenceEqual(tokens_r['input_ids']    ,[0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]			)

																	self.assertSequenceEqual(
																	    lowerCamelCase__    ,['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']			)
																	self.assertSequenceEqual(
																	    lowerCamelCase__    ,['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']			)








'''simple docstring'''





from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


_lowerCAmelCase      						=	'''\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
'''

_lowerCAmelCase      						=	'''\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
'''

_lowerCAmelCase      						=	'''
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for \'record\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'prediction_text\': the predicted answer text
        - for \'multirc\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question-answer pair as specified by the dataset
            - \'prediction\': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for \'record\': list of question-answers dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'answers\': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for \'record\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1\': F1 score
    - for \'multirc\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1_m\': Per-question macro-F1 score
        - \'f1_a\': Average F1 score over all answers
    - for \'axb\':
        \'matthews_correlation\': Matthew Correlation
    - for \'cb\':
        - \'accuracy\': Accuracy
        - \'f1\': F1 score
    - for all others:
        - \'accuracy\': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')
    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]
    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')
    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'matthews_correlation\': 1.0}
'''
def    _SCREAMING_SNAKE_CASE  (       UpperCamelCase							,    UpperCamelCase			):



    """simple docstring"""





    return float((preds == labels).mean()			)
def    _SCREAMING_SNAKE_CASE  (       UpperCamelCase							,    UpperCamelCase							,    UpperCamelCase="binary"			):



    """simple docstring"""





    lowerCAmelCase__ :  Any							      =					simple_accuracy(UpperCamelCase							,    UpperCamelCase			)
    lowerCAmelCase__ :  Tuple							      =					float(fa_score(y_true=UpperCamelCase							,    y_pred=UpperCamelCase							,    average=UpperCamelCase			)			)
    return {
        "accuracy": acc,
        "f1": fa,
    }
def    _SCREAMING_SNAKE_CASE  (       UpperCamelCase							,    UpperCamelCase			):



    """simple docstring"""





    lowerCAmelCase__ :  List[str]							      =					{}
    for id_pred, label in zip(UpperCamelCase							,    UpperCamelCase			):
        lowerCAmelCase__ :  str							      =					f"""{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}"""
        lowerCAmelCase__ :  Dict							      =					id_pred["""prediction"""]
        if question_id in question_map:
            question_map[question_id].append((pred, label)			)
        else:
            lowerCAmelCase__ :  Optional[int]							      =					[(pred, label)]
    lowerCAmelCase__      ,   lowerCAmelCase__ :  int							      =					[], []
    for question, preds_labels in question_map.items():
        lowerCAmelCase__      ,   lowerCAmelCase__ :  Optional[int]							      =					zip(*UpperCamelCase			)
        lowerCAmelCase__ :  List[Any]							      =					fa_score(y_true=UpperCamelCase							,    y_pred=UpperCamelCase							,    average="""macro"""			)
        fas.append(UpperCamelCase			)
        lowerCAmelCase__ :  Union[str, Any]							      =					int(sum(pred == label for pred, label in preds_labels			) == len(UpperCamelCase			)			)
        ems.append(UpperCamelCase			)
    lowerCAmelCase__ :  Optional[Any]							      =					float(sum(UpperCamelCase			) / len(UpperCamelCase			)			)
    lowerCAmelCase__ :  List[Any]							      =					sum(UpperCamelCase			) / len(UpperCamelCase			)
    lowerCAmelCase__ :  Dict							      =					float(fa_score(y_true=UpperCamelCase							,    y_pred=[id_pred["""prediction"""] for id_pred in ids_preds]			)			)
    return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION						, _KWARGS_DESCRIPTION     )
class        lowerCAmelCase_(				datasets.Metric     ):
  '''simple docstring'''




  def 							UpperCAmelCase_	(    self      )    ->		Optional[Any]:
      if self.config_name not in [
          "boolq",
          "cb",
          "copa",
          "multirc",
          "record",
          "rte",
          "wic",
          "wsc",
          "wsc.fixed",
          "axb",
          "axg",
      ]:
          raise KeyError(
              """You should supply a configuration name selected in """
              """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"""      )
      return datasets.MetricInfo(
          description=_DESCRIPTION  ,citation=_CITATION  ,inputs_description=_KWARGS_DESCRIPTION  ,features=datasets.Features(self._get_feature_types()      )  ,codebase_urls=[]  ,reference_urls=[]  ,format="""numpy""" if not self.config_name == """record""" and not self.config_name == """multirc""" else None  ,)




  def 							UpperCAmelCase_	(    self      )    ->		str:
      if self.config_name == "record":
          return {
              "predictions": {
                  "idx": {
                      "passage": datasets.Value("""int64"""      ),
                      "query": datasets.Value("""int64"""      ),
                  },
                  "prediction_text": datasets.Value("""string"""      ),
              },
              "references": {
                  "idx": {
                      "passage": datasets.Value("""int64"""      ),
                      "query": datasets.Value("""int64"""      ),
                  },
                  "answers": datasets.Sequence(datasets.Value("""string"""      )      ),
              },
          }
      elif self.config_name == "multirc":
          return {
              "predictions": {
                  "idx": {
                      "answer": datasets.Value("""int64"""      ),
                      "paragraph": datasets.Value("""int64"""      ),
                      "question": datasets.Value("""int64"""      ),
                  },
                  "prediction": datasets.Value("""int64"""      ),
              },
              "references": datasets.Value("""int64"""      ),
          }
      else:
          return {
              "predictions": datasets.Value("""int64"""      ),
              "references": datasets.Value("""int64"""      ),
          }






  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase      )    ->		Any:
      if self.config_name == "axb":
          return {"matthews_correlation": matthews_corrcoef(__UpperCAmelCase  ,__UpperCAmelCase      )}
      elif self.config_name == "cb":
          return acc_and_fa(__UpperCAmelCase  ,__UpperCAmelCase  ,fa_avg="""macro"""      )
      elif self.config_name == "record":
          lowerCAmelCase__ :  Optional[Any]							      =					[
              {
                  """qas""": [
                      {"""id""": ref["""idx"""]["""query"""], """answers""": [{"""text""": ans} for ans in ref["""answers"""]]}
                      for ref in references
                  ]
              }
          ]
          lowerCAmelCase__ :  Union[str, Any]							      =					{pred["""idx"""]["""query"""]: pred["""prediction_text"""] for pred in predictions}
          return evaluate_record(__UpperCAmelCase  ,__UpperCAmelCase      )[0]
      elif self.config_name == "multirc":
          return evaluate_multirc(__UpperCAmelCase  ,__UpperCAmelCase      )
      elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
          return {"accuracy": simple_accuracy(__UpperCAmelCase  ,__UpperCAmelCase      )}
      else:
          raise KeyError(
              """You should supply a configuration name selected in """
              """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"""      )


