
import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
  lowercase  :					Tuple						    =							"""pt"""
elif is_tf_available():
  lowercase  :					List[str]						    =							"""tf"""
else:
  lowercase  :					str						    =							"""jax"""

class __snake_case		(      lowerCAmelCase					,	unittest.TestCase     ):
    _a		:						List[str]=				ByTaTokenizer
    _a		:						List[str]=				False
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           super().setUp()
           lowercase							:     Optional[Any]       	=      ByTaTokenizer()
           tokenizer.save_pretrained(self.tmpdirname      )
    @cached_property
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           return ByTaTokenizer.from_pretrained("""google/byt5-small"""      )
    def       _SCREAMING_SNAKE_CASE       (  self		,**snake_case      ):



           '''simple docstring'''







           return self.tokenizer_class.from_pretrained(self.tmpdirname		,**snake_case      )
    def       _SCREAMING_SNAKE_CASE       (  self		,snake_case		,snake_case=False		,snake_case=20		,snake_case=5      ):



           '''simple docstring'''







           lowercase							:     Optional[int]       	=      []
           for i in range(len(snake_case      )      ):
                  try:
                         lowercase							:     Union[str, Any]       	=      tokenizer.decode([i]		,clean_up_tokenization_spaces=snake_case      )
                  except UnicodeDecodeError:
                         pass
                  toks.append((i, tok)      )

           lowercase							:     Optional[int]       	=      list(filter(lambda snake_case      : re.match(r"""^[ a-zA-Z]+$"""		,t[1]      )		,snake_case      )      )
           lowercase							:     int       	=      list(filter(lambda snake_case      : [t[0]] == tokenizer.encode(t[1]		,add_special_tokens=snake_case      )		,snake_case      )      )
           if max_length is not None and len(snake_case      ) > max_length:
                  lowercase							:     Optional[Any]       	=      toks[:max_length]
           if min_length is not None and len(snake_case      ) < min_length and len(snake_case      ) > 0:
                  while len(snake_case      ) < min_length:
                         lowercase							:     Optional[Any]       	=      toks + toks
        # toks_str = [t[1] for t in toks]
           lowercase							:     Dict       	=      [t[0] for t in toks]

           # Ensure consistency
           lowercase							:     int       	=      tokenizer.decode(snake_case		,clean_up_tokenization_spaces=snake_case      )
           if " " not in output_txt and len(snake_case      ) > 1:
                  lowercase							:     int       	=      (
                      tokenizer.decode([toks_ids[0]]		,clean_up_tokenization_spaces=snake_case      )
                      + """ """
                      + tokenizer.decode(toks_ids[1:]		,clean_up_tokenization_spaces=snake_case      )
                  )
           if with_prefix_space:
                  lowercase							:     Optional[int]       	=      """ """ + output_txt
           lowercase							:     int       	=      tokenizer.encode(snake_case		,add_special_tokens=snake_case      )
           return output_txt, output_ids
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     List[str]       	=      self.ta_base_tokenizer
           lowercase							:     Tuple       	=      tokenizer(["""hi</s>""", """I went to the gym</s>""", """</s>"""]      )
           lowercase							:     List[str]       	=      tokenizer(["""hi""", """I went to the gym""", """"""]      )
           self.assertListEqual(batch_with_eos_added["""input_ids"""]		,batch_without_eos_added["""input_ids"""]      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Tuple       	=      self.ta_base_tokenizer
           lowercase							:     Union[str, Any]       	=      """Unicode €."""
           lowercase							:     int       	=      tokenizer(snake_case      )
           lowercase							:     Optional[int]       	=      [88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
           self.assertEqual(encoded["""input_ids"""]		,snake_case      )

           # decoding
           lowercase							:     str       	=      tokenizer.decode(snake_case      )
           self.assertEqual(snake_case		,"""Unicode €.</s>"""      )

           lowercase							:     str       	=      tokenizer("""e è é ê ë"""      )
           lowercase							:     Any       	=      [104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
           self.assertEqual(encoded["""input_ids"""]		,snake_case      )
           # decoding
           lowercase							:     str       	=      tokenizer.decode(snake_case      )
           self.assertEqual(snake_case		,"""e è é ê ë</s>"""      )

           # encode/decode, but with `encode` instead of `__call__`
           self.assertEqual(tokenizer.decode(tokenizer.encode("""e è é ê ë"""      )      )		,"""e è é ê ë</s>"""      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     List[Any]       	=      self.ta_base_tokenizer
           lowercase							:     Optional[int]       	=      ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
           # fmt: off
           lowercase							:     str       	=      [68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
           # fmt: on
           lowercase							:     Optional[Any]       	=      tokenizer(snake_case		,padding=snake_case		,return_tensors=snake_case      )
           self.assertIsInstance(snake_case		,snake_case      )

           if FRAMEWORK != "jax":
                  lowercase							:     int       	=      list(batch.input_ids.numpy()[0]      )
           else:
                  lowercase							:     Optional[int]       	=      list(batch.input_ids.tolist()[0]      )

           self.assertListEqual(snake_case		,snake_case      )

           self.assertEqual((2, 37)		,batch.input_ids.shape      )
           self.assertEqual((2, 37)		,batch.attention_mask.shape      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     List[str]       	=      self.ta_base_tokenizer
           lowercase							:     Optional[int]       	=      ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
           lowercase							:     str       	=      tokenizer(snake_case		,padding=snake_case		,return_tensors=snake_case      )
           # check if input_ids are returned and no decoder_input_ids
           self.assertIn("""input_ids"""		,snake_case      )
           self.assertIn("""attention_mask"""		,snake_case      )
           self.assertNotIn("""decoder_input_ids"""		,snake_case      )
           self.assertNotIn("""decoder_attention_mask"""		,snake_case      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Any       	=      self.ta_base_tokenizer
           lowercase							:     Tuple       	=      [
               """Summary of the text.""",
               """Another summary.""",
           ]
           lowercase							:     List[str]       	=      tokenizer(
               text_target=snake_case		,max_length=32		,padding="""max_length"""		,truncation=snake_case		,return_tensors=snake_case      )
           self.assertEqual(32		,targets["""input_ids"""].shape[1]      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     List[Any]       	=      self.ta_base_tokenizer
           lowercase							:     str       	=      ["""A long paragraph for summarization. </s>"""]
           lowercase							:     Optional[Any]       	=      ["""Summary of the text. </s>"""]
           # fmt: off
           lowercase							:     List[Any]       	=      [68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
           lowercase							:     str       	=      [86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
           # fmt: on

           lowercase							:     int       	=      tokenizer(snake_case		,text_target=snake_case      )

           self.assertEqual(snake_case		,batch["""input_ids"""][0]      )
           self.assertEqual(snake_case		,batch["""labels"""][0]      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     List[Any]       	=      self.get_tokenizers()
           for tokenizer in tokenizers:
                  with self.subTest(f"{tokenizer.__class__.__name__}"      ):
                         self.assertNotEqual(tokenizer.model_max_length		,42      )

        # Now let's start the test
           lowercase							:     Any       	=      self.get_tokenizers()
           for tokenizer in tokenizers:
                  with self.subTest(f"{tokenizer.__class__.__name__}"      ):
                         # Isolate this from the other tests because we save additional tokens/etc
                         lowercase							:     int       	=      tempfile.mkdtemp()

                         lowercase							:     Optional[int]       	=      """ He is very happy, UNwant\u00E9d,running"""
                         lowercase							:     Any       	=      tokenizer.encode(snake_case		,add_special_tokens=snake_case      )
                         tokenizer.save_pretrained(snake_case      )

                         lowercase							:     str       	=      tokenizer.__class__.from_pretrained(snake_case      )
                         lowercase							:     Optional[int]       	=      after_tokenizer.encode(snake_case		,add_special_tokens=snake_case      )
                         self.assertListEqual(snake_case		,snake_case      )

                         shutil.rmtree(snake_case      )

           lowercase							:     str       	=      self.get_tokenizers(model_max_length=42      )
           for tokenizer in tokenizers:
                  with self.subTest(f"{tokenizer.__class__.__name__}"      ):
                         # Isolate this from the other tests because we save additional tokens/etc
                         lowercase							:     Tuple       	=      tempfile.mkdtemp()

                         lowercase							:     str       	=      """ He is very happy, UNwant\u00E9d,running"""
                         tokenizer.add_tokens(["""bim""", """bambam"""]      )
                         lowercase							:     List[str]       	=      tokenizer.additional_special_tokens
                         additional_special_tokens.append("""new_additional_special_token"""      )
                         tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}      )
                         lowercase							:     Any       	=      tokenizer.encode(snake_case		,add_special_tokens=snake_case      )
                         tokenizer.save_pretrained(snake_case      )

                         lowercase							:     Optional[int]       	=      tokenizer.__class__.from_pretrained(snake_case      )
                         lowercase							:     Union[str, Any]       	=      after_tokenizer.encode(snake_case		,add_special_tokens=snake_case      )
                         self.assertListEqual(snake_case		,snake_case      )
                         self.assertIn("""new_additional_special_token"""		,after_tokenizer.additional_special_tokens      )
                         self.assertEqual(after_tokenizer.model_max_length		,42      )

                         lowercase							:     Union[str, Any]       	=      tokenizer.__class__.from_pretrained(snake_case		,model_max_length=43      )
                         self.assertEqual(tokenizer.model_max_length		,43      )

                         shutil.rmtree(snake_case      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     str       	=      []
           if self.test_slow_tokenizer:
                  tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())      )

           if self.test_rust_tokenizer:
                  tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())      )

           for tokenizer_class, tokenizer_utils in tokenizer_list:
                  with tempfile.TemporaryDirectory() as tmp_dir:
                         tokenizer_utils.save_pretrained(snake_case      )

                         with open(os.path.join(snake_case		,"""special_tokens_map.json"""      )		,encoding="""utf-8"""      ) as json_file:
                                lowercase							:     Dict       	=      json.load(snake_case      )

                         with open(os.path.join(snake_case		,"""tokenizer_config.json"""      )		,encoding="""utf-8"""      ) as json_file:
                                lowercase							:     List[Any]       	=      json.load(snake_case      )

                         lowercase							:     List[Any]       	=      [f"<extra_id_{i}>" for i in range(125      )]

                         lowercase							:     List[Any]       	=      added_tokens_extra_ids + [
                             """an_additional_special_token"""
                         ]
                         lowercase							:     Optional[int]       	=      added_tokens_extra_ids + [
                             """an_additional_special_token"""
                         ]

                         with open(os.path.join(snake_case		,"""special_tokens_map.json"""      )		,"""w"""		,encoding="""utf-8"""      ) as outfile:
                                json.dump(snake_case		,snake_case      )
                         with open(os.path.join(snake_case		,"""tokenizer_config.json"""      )		,"""w"""		,encoding="""utf-8"""      ) as outfile:
                                json.dump(snake_case		,snake_case      )

                         # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                         # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                         # "special_tokens_map.json" files
                         lowercase							:     Any       	=      tokenizer_class.from_pretrained(
                             snake_case		,)
                         self.assertIn(
                             """an_additional_special_token"""		,tokenizer_without_change_in_init.additional_special_tokens      )
                         # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                         self.assertEqual(
                             ["""an_additional_special_token"""]		,tokenizer_without_change_in_init.convert_ids_to_tokens(
                                 tokenizer_without_change_in_init.convert_tokens_to_ids(["""an_additional_special_token"""]      )      )		,)

                         # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                         lowercase							:     Union[str, Any]       	=      added_tokens_extra_ids + [AddedToken("""a_new_additional_special_token"""		,lstrip=snake_case      )]
                         lowercase							:     Union[str, Any]       	=      tokenizer_class.from_pretrained(
                             snake_case		,additional_special_tokens=snake_case		,)

                         self.assertIn("""a_new_additional_special_token"""		,tokenizer.additional_special_tokens      )
                         self.assertEqual(
                             ["""a_new_additional_special_token"""]		,tokenizer.convert_ids_to_tokens(
                                 tokenizer.convert_tokens_to_ids(["""a_new_additional_special_token"""]      )      )		,)
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Optional[int]       	=      []
           if self.test_slow_tokenizer:
                  tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())      )

           if self.test_rust_tokenizer:
                  tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())      )

           for tokenizer_class, tokenizer_utils in tokenizer_list:
                  with tempfile.TemporaryDirectory() as tmp_dir:
                         tokenizer_utils.save_pretrained(snake_case      )

                         lowercase							:     Any       	=      tokenizer_class.from_pretrained(snake_case      )

                         self.assertTrue(tokenizer.decode([255]      ) == """"""      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           pass
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           pass
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           pass
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           pass
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     int       	=      self.get_tokenizers(fast=snake_case		,do_lower_case=snake_case      )
           for tokenizer in tokenizers:
                  with self.subTest(f"{tokenizer.__class__.__name__}"      ):
                         lowercase							:     Dict       	=      ["""t""", """h""", """i""", """s""", """ """, """i""", """s""", """ """, """a""", """ """, """t""", """e""", """x""", """t""", """</s>"""]
                         lowercase							:     Any       	=      tokenizer.convert_tokens_to_string(snake_case      )

                         self.assertIsInstance(snake_case		,snake_case      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     int       	=      self.get_tokenizers()
           for tokenizer in tokenizers:
                  with self.subTest(f"{tokenizer.__class__.__name__}"      ):
                         lowercase							:     Optional[Any]       	=      [
                             """bos_token""",
                             """eos_token""",
                             """unk_token""",
                             """sep_token""",
                             """pad_token""",
                             """cls_token""",
                             """mask_token""",
                         ]

                         lowercase							:     str       	=      0
                         lowercase							:     int       	=      tokenizer.convert_ids_to_tokens(
                             snake_case		,skip_special_tokens=snake_case      )

                         for attr in attributes_list:
                                setattr(snake_case		,attr + """_id"""		,snake_case      )
                                self.assertEqual(getattr(snake_case		,snake_case      )		,snake_case      )
                                self.assertEqual(getattr(snake_case		,attr + """_id"""      )		,snake_case      )

                                setattr(snake_case		,attr + """_id"""		,snake_case      )
                                self.assertEqual(getattr(snake_case		,snake_case      )		,snake_case      )
                                self.assertEqual(getattr(snake_case		,attr + """_id"""      )		,snake_case      )

                         setattr(snake_case		,"""additional_special_tokens_ids"""		,[]      )
                         self.assertListEqual(getattr(snake_case		,"""additional_special_tokens"""      )		,[]      )
                         self.assertListEqual(getattr(snake_case		,"""additional_special_tokens_ids"""      )		,[]      )

                         setattr(snake_case		,"""additional_special_tokens_ids"""		,[token_id_to_test_setters]      )
                         self.assertListEqual(getattr(snake_case		,"""additional_special_tokens"""      )		,[token_to_test_setters]      )
                         self.assertListEqual(getattr(snake_case		,"""additional_special_tokens_ids"""      )		,[token_id_to_test_setters]      )


from scipy.stats import pearsonr, spearmanr
from sklearn.metrics import fa_score, matthews_corrcoef

import datasets


lowercase  :					str						    =							"""\
@inproceedings{wang2019glue,
  title={{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding},
  author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R.},
  note={In the Proceedings of ICLR.},
  year={2019}
}
"""

lowercase  :					Dict						    =							"""\
GLUE, the General Language Understanding Evaluation benchmark
(https://gluebenchmark.com/) is a collection of resources for training,
evaluating, and analyzing natural language understanding systems.
"""

lowercase  :					int						    =							"""
Compute GLUE evaluation metric associated to each GLUE dataset.
Args:
    predictions: list of predictions to score.
        Each translation should be tokenized into a list of tokens.
    references: list of lists of references for each translation.
        Each reference should be tokenized into a list of tokens.
Returns: depending on the GLUE subset, one or several of:
    \"accuracy\": Accuracy
    \"f1\": F1 score
    \"pearson\": Pearson Correlation
    \"spearmanr\": Spearman Correlation
    \"matthews_correlation\": Matthew Correlation
Examples:

    >>> glue_metric = datasets.load_metric('glue', 'sst2')  # 'sst2' or any of [\"mnli\", \"mnli_mismatched\", \"mnli_matched\", \"qnli\", \"rte\", \"wnli\", \"hans\"]
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}

    >>> glue_metric = datasets.load_metric('glue', 'mrpc')  # 'mrpc' or 'qqp'
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0, 'f1': 1.0}

    >>> glue_metric = datasets.load_metric('glue', 'stsb')
    >>> references = [0., 1., 2., 3., 4., 5.]
    >>> predictions = [0., 1., 2., 3., 4., 5.]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print({\"pearson\": round(results[\"pearson\"], 2), \"spearmanr\": round(results[\"spearmanr\"], 2)})
    {'pearson': 1.0, 'spearmanr': 1.0}

    >>> glue_metric = datasets.load_metric('glue', 'cola')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'matthews_correlation': 1.0}
"""







def 						_snake_case(  SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__				) ->    Optional[int]:
       return float((preds == labels).mean()				)







def 						_snake_case(  SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__				) ->    Optional[Any]:
       lowercase							:     Any       	=      simple_accuracy(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__				)
       lowercase							:     Union[str, Any]       	=      float(fa_score(y_true=SCREAMING_SNAKE_CASE__					,	y_pred=SCREAMING_SNAKE_CASE__				)				)
       return {
           "accuracy": acc,
           "f1": fa,
       }







def 						_snake_case(  SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__				) ->    List[Any]:
       lowercase							:     Union[str, Any]       	=      float(pearsonr(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__				)[0]				)
       lowercase							:     Dict       	=      float(spearmanr(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__				)[0]				)
       return {
           "pearson": pearson_corr,
           "spearmanr": spearman_corr,
       }




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION					,	_KWARGS_DESCRIPTION     )
class __snake_case		(      datasets.Metric     ):
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           if self.config_name not in [
               "sst2",
               "mnli",
               "mnli_mismatched",
               "mnli_matched",
               "cola",
               "stsb",
               "mrpc",
               "qqp",
               "qnli",
               "rte",
               "wnli",
               "hans",
           ]:
                  raise KeyError(
                      """You should supply a configuration name selected in """
                      """[\"sst2\", \"mnli\", \"mnli_mismatched\", \"mnli_matched\", """
                      """\"cola\", \"stsb\", \"mrpc\", \"qqp\", \"qnli\", \"rte\", \"wnli\", \"hans\"]"""      )
           return datasets.MetricInfo(
               description=_DESCRIPTION		,citation=_CITATION		,inputs_description=_KWARGS_DESCRIPTION		,features=datasets.Features(
                   {
                       """predictions""": datasets.Value("""int64""" if self.config_name != """stsb""" else """float32"""      ),
                       """references""": datasets.Value("""int64""" if self.config_name != """stsb""" else """float32"""      ),
                   }      )		,codebase_urls=[]		,reference_urls=[]		,format="""numpy"""		,)
    def       _SCREAMING_SNAKE_CASE       (  self		,snake_case		,snake_case      ):



           '''simple docstring'''







           if self.config_name == "cola":
                  return {"matthews_correlation": matthews_corrcoef(snake_case		,snake_case      )}
           elif self.config_name == "stsb":
                  return pearson_and_spearman(snake_case		,snake_case      )
           elif self.config_name in ["mrpc", "qqp"]:
                  return acc_and_fa(snake_case		,snake_case      )
           elif self.config_name in ["sst2", "mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]:
                  return {"accuracy": simple_accuracy(snake_case		,snake_case      )}
           else:
                  raise KeyError(
                      """You should supply a configuration name selected in """
                      """[\"sst2\", \"mnli\", \"mnli_mismatched\", \"mnli_matched\", """
                      """\"cola\", \"stsb\", \"mrpc\", \"qqp\", \"qnli\", \"rte\", \"wnli\", \"hans\"]"""      )
