
"""simple docstring"""




import unittest

import numpy as np
from datasets import load_dataset

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
					import torch

if is_vision_available():
					from PIL import Image

					from transformers import BeitImageProcessor



class 	lowerCAmelCase_						(unittest.TestCase  ):


				"""simple docstring"""



				def __init__(self   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__=7   ,       SCREAMING_SNAKE_CASE__=3   ,       SCREAMING_SNAKE_CASE__=18   ,       SCREAMING_SNAKE_CASE__=30   ,       SCREAMING_SNAKE_CASE__=4_00   ,       SCREAMING_SNAKE_CASE__=True   ,       SCREAMING_SNAKE_CASE__=None   ,       SCREAMING_SNAKE_CASE__=True   ,       SCREAMING_SNAKE_CASE__=None   ,       SCREAMING_SNAKE_CASE__=True   ,       SCREAMING_SNAKE_CASE__=[0.5, 0.5, 0.5]   ,       SCREAMING_SNAKE_CASE__=[0.5, 0.5, 0.5]   ,       SCREAMING_SNAKE_CASE__=False   ,       )					->		Union[str, Any]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   size if size is not None else {"""height""": 20, """width""": 20}
											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   crop_size if crop_size is not None else {"""height""": 18, """width""": 18}
											SCREAMING_SNAKE_CASE__						:       int           =   parent
											SCREAMING_SNAKE_CASE__						:       Any           =   batch_size
											SCREAMING_SNAKE_CASE__						:       Tuple           =   num_channels
											SCREAMING_SNAKE_CASE__						:       Optional[int]           =   image_size
											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   min_resolution
											SCREAMING_SNAKE_CASE__						:       int           =   max_resolution
											SCREAMING_SNAKE_CASE__						:       Dict           =   do_resize
											SCREAMING_SNAKE_CASE__						:       Any           =   size
											SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   do_center_crop
											SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   crop_size
											SCREAMING_SNAKE_CASE__						:       Any           =   do_normalize
											SCREAMING_SNAKE_CASE__						:       str           =   image_mean
											SCREAMING_SNAKE_CASE__						:       str           =   image_std
											SCREAMING_SNAKE_CASE__						:       Any           =   do_reduce_labels



				def 	__magic_name__    (self    )					->		List[str]:



											"""simple docstring"""




											return {
											    "do_resize": self.do_resize,
											    "size": self.size,
											    "do_center_crop": self.do_center_crop,
											    "crop_size": self.crop_size,
											    "do_normalize": self.do_normalize,
											    "image_mean": self.image_mean,
											    "image_std": self.image_std,
											    "do_reduce_labels": self.do_reduce_labels,
											}






def lowercase_  (   ):
							SCREAMING_SNAKE_CASE__						:       Any           =   load_dataset("""hf-internal-testing/fixtures_ade20k"""   ,split="""test"""						)

							SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   Image.open(dataset[0]["""file"""]						)
							SCREAMING_SNAKE_CASE__						:       int           =   Image.open(dataset[1]["""file"""]						)

							return image, map

def lowercase_  (   ):
							SCREAMING_SNAKE_CASE__						:       str           =   load_dataset("""hf-internal-testing/fixtures_ade20k"""   ,split="""test"""						)

							SCREAMING_SNAKE_CASE__						:       Dict           =   Image.open(ds[0]["""file"""]						)
							SCREAMING_SNAKE_CASE__						:       Tuple           =   Image.open(ds[1]["""file"""]						)
							SCREAMING_SNAKE_CASE__						:       List[str]           =   Image.open(ds[2]["""file"""]						)
							SCREAMING_SNAKE_CASE__						:       Tuple           =   Image.open(ds[3]["""file"""]						)

							return [imagea, imagea], [mapa, mapa]



@require_torch
@require_vision
class 	lowerCAmelCase_						(a__ ,					unittest.TestCase  ):


				"""simple docstring"""



				__UpperCamelCase      :						Union[str, Any]					       =   BeitImageProcessor if is_vision_available() else None



				def 	__magic_name__    (self    )					->		str:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Optional[int]           =   BeitImageProcessingTester(self    )



				@property
				def 	__magic_name__    (self    )					->		Optional[int]:



											"""simple docstring"""




											return self.image_processor_tester.prepare_image_processor_dict()



				def 	__magic_name__    (self    )					->		Optional[Any]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   self.image_processing_class(**self.image_processor_dict    )
											self.assertTrue(hasattr(SCREAMING_SNAKE_CASE__   ,       """do_resize"""    )    )
											self.assertTrue(hasattr(SCREAMING_SNAKE_CASE__   ,       """size"""    )    )
											self.assertTrue(hasattr(SCREAMING_SNAKE_CASE__   ,       """do_center_crop"""    )    )
											self.assertTrue(hasattr(SCREAMING_SNAKE_CASE__   ,       """center_crop"""    )    )
											self.assertTrue(hasattr(SCREAMING_SNAKE_CASE__   ,       """do_normalize"""    )    )
											self.assertTrue(hasattr(SCREAMING_SNAKE_CASE__   ,       """image_mean"""    )    )
											self.assertTrue(hasattr(SCREAMING_SNAKE_CASE__   ,       """image_std"""    )    )



				def 	__magic_name__    (self    )					->		str:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       List[Any]           =   self.image_processing_class.from_dict(self.image_processor_dict    )
											self.assertEqual(image_processor.size   ,       {"""height""": 20, """width""": 20}    )
											self.assertEqual(image_processor.crop_size   ,       {"""height""": 18, """width""": 18}    )
											self.assertEqual(image_processor.do_reduce_labels   ,       SCREAMING_SNAKE_CASE__    )

											SCREAMING_SNAKE_CASE__						:       Optional[int]           =   self.image_processing_class.from_dict(
											    self.image_processor_dict   ,       size=42   ,       crop_size=84   ,       reduce_labels=SCREAMING_SNAKE_CASE__    )
											self.assertEqual(image_processor.size   ,       {"""height""": 42, """width""": 42}    )
											self.assertEqual(image_processor.crop_size   ,       {"""height""": 84, """width""": 84}    )
											self.assertEqual(image_processor.do_reduce_labels   ,       SCREAMING_SNAKE_CASE__    )



				def 	__magic_name__    (self    )					->		Optional[int]:



											"""simple docstring"""




											pass



				def 	__magic_name__    (self    )					->		List[Any]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       int           =   self.image_processing_class(**self.image_processor_dict    )
											# create random PIL images
											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   prepare_image_inputs(self.image_processor_tester   ,       equal_resolution=SCREAMING_SNAKE_CASE__    )
											for image in image_inputs:
																		self.assertIsInstance(SCREAMING_SNAKE_CASE__   ,       Image.Image    )

											# Test not batched input
											SCREAMING_SNAKE_CASE__						:       Tuple           =   image_processing(image_inputs[0]   ,       return_tensors="""pt"""    ).pixel_values
											self.assertEqual(
											    encoded_images.shape   ,       (
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )   ,       )

											# Test batched
											SCREAMING_SNAKE_CASE__						:       int           =   image_processing(SCREAMING_SNAKE_CASE__   ,       return_tensors="""pt"""    ).pixel_values
											self.assertEqual(
											    encoded_images.shape   ,       (
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )   ,       )



				def 	__magic_name__    (self    )					->		Union[str, Any]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Optional[int]           =   self.image_processing_class(**self.image_processor_dict    )
											# create random numpy tensors
											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   prepare_image_inputs(self.image_processor_tester   ,       equal_resolution=SCREAMING_SNAKE_CASE__   ,       numpify=SCREAMING_SNAKE_CASE__    )
											for image in image_inputs:
																		self.assertIsInstance(SCREAMING_SNAKE_CASE__   ,       np.ndarray    )

											# Test not batched input
											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   image_processing(image_inputs[0]   ,       return_tensors="""pt"""    ).pixel_values
											self.assertEqual(
											    encoded_images.shape   ,       (
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )   ,       )

											# Test batched
											SCREAMING_SNAKE_CASE__						:       Any           =   image_processing(SCREAMING_SNAKE_CASE__   ,       return_tensors="""pt"""    ).pixel_values
											self.assertEqual(
											    encoded_images.shape   ,       (
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )   ,       )



				def 	__magic_name__    (self    )					->		Optional[Any]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Tuple           =   self.image_processing_class(**self.image_processor_dict    )
											# create random PyTorch tensors
											SCREAMING_SNAKE_CASE__						:       Tuple           =   prepare_image_inputs(self.image_processor_tester   ,       equal_resolution=SCREAMING_SNAKE_CASE__   ,       torchify=SCREAMING_SNAKE_CASE__    )
											for image in image_inputs:
																		self.assertIsInstance(SCREAMING_SNAKE_CASE__   ,       torch.Tensor    )

											# Test not batched input
											SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   image_processing(image_inputs[0]   ,       return_tensors="""pt"""    ).pixel_values
											self.assertEqual(
											    encoded_images.shape   ,       (
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )   ,       )

											# Test batched
											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   image_processing(SCREAMING_SNAKE_CASE__   ,       return_tensors="""pt"""    ).pixel_values
											self.assertEqual(
											    encoded_images.shape   ,       (
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )   ,       )



				def 	__magic_name__    (self    )					->		Tuple:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       str           =   self.image_processing_class(**self.image_processor_dict    )
											# create random PyTorch tensors
											SCREAMING_SNAKE_CASE__						:       List[str]           =   prepare_image_inputs(self.image_processor_tester   ,       equal_resolution=SCREAMING_SNAKE_CASE__   ,       torchify=SCREAMING_SNAKE_CASE__    )
											SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   []
											for image in image_inputs:
																		self.assertIsInstance(SCREAMING_SNAKE_CASE__   ,       torch.Tensor    )
																		maps.append(torch.zeros(image.shape[-2:]    ).long()    )

											# Test not batched input
											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   image_processing(image_inputs[0]   ,       maps[0]   ,       return_tensors="""pt"""    )
											self.assertEqual(
											    encoding["""pixel_values"""].shape   ,       (
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )   ,       )
											self.assertEqual(
											    encoding["""labels"""].shape   ,       (
											        1,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )   ,       )
											self.assertEqual(encoding["""labels"""].dtype   ,       torch.long    )
											self.assertTrue(encoding["""labels"""].min().item() >= 0    )
											self.assertTrue(encoding["""labels"""].max().item() <= 2_55    )

											# Test batched
											SCREAMING_SNAKE_CASE__						:       Optional[int]           =   image_processing(SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__   ,       return_tensors="""pt"""    )
											self.assertEqual(
											    encoding["""pixel_values"""].shape   ,       (
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )   ,       )
											self.assertEqual(
											    encoding["""labels"""].shape   ,       (
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )   ,       )
											self.assertEqual(encoding["""labels"""].dtype   ,       torch.long    )
											self.assertTrue(encoding["""labels"""].min().item() >= 0    )
											self.assertTrue(encoding["""labels"""].max().item() <= 2_55    )

											# Test not batched input (PIL images)
											SCREAMING_SNAKE_CASE__      ,      SCREAMING_SNAKE_CASE__						:       str           =   prepare_semantic_single_inputs()

											SCREAMING_SNAKE_CASE__						:       Dict           =   image_processing(SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__   ,       return_tensors="""pt"""    )
											self.assertEqual(
											    encoding["""pixel_values"""].shape   ,       (
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )   ,       )
											self.assertEqual(
											    encoding["""labels"""].shape   ,       (
											        1,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )   ,       )
											self.assertEqual(encoding["""labels"""].dtype   ,       torch.long    )
											self.assertTrue(encoding["""labels"""].min().item() >= 0    )
											self.assertTrue(encoding["""labels"""].max().item() <= 2_55    )

											# Test batched input (PIL images)
											SCREAMING_SNAKE_CASE__      ,      SCREAMING_SNAKE_CASE__						:       str           =   prepare_semantic_batch_inputs()

											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   image_processing(SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__   ,       return_tensors="""pt"""    )
											self.assertEqual(
											    encoding["""pixel_values"""].shape   ,       (
											        2,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )   ,       )
											self.assertEqual(
											    encoding["""labels"""].shape   ,       (
											        2,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )   ,       )
											self.assertEqual(encoding["""labels"""].dtype   ,       torch.long    )
											self.assertTrue(encoding["""labels"""].min().item() >= 0    )
											self.assertTrue(encoding["""labels"""].max().item() <= 2_55    )



				def 	__magic_name__    (self    )					->		Dict:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Any           =   self.image_processing_class(**self.image_processor_dict    )

											# ADE20k has 150 classes, and the background is included, so labels should be between 0 and 150
											SCREAMING_SNAKE_CASE__      ,      SCREAMING_SNAKE_CASE__						:       Dict           =   prepare_semantic_single_inputs()
											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   image_processing(SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__   ,       return_tensors="""pt"""    )
											self.assertTrue(encoding["""labels"""].min().item() >= 0    )
											self.assertTrue(encoding["""labels"""].max().item() <= 1_50    )

											SCREAMING_SNAKE_CASE__						:       List[Any]           =   True
											SCREAMING_SNAKE_CASE__						:       int           =   image_processing(SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__   ,       return_tensors="""pt"""    )
											self.assertTrue(encoding["""labels"""].min().item() >= 0    )
											self.assertTrue(encoding["""labels"""].max().item() <= 2_55    )







'''simple docstring'''
import numpy as np
from scipy.spatial.distance import cdist
from sklearn.metrics import fa_score

import datasets


lowerCamelCase :      List[Any]					 =     "\\n    @inproceedings{kakwani2020indicnlpsuite,\n    title={{IndicNLPSuite: Monolingual Corpora, Evaluation Benchmarks and Pre-trained Multilingual Language Models for Indian Languages}},\n    author={Divyanshu Kakwani and Anoop Kunchukuttan and Satish Golla and Gokul N.C. and Avik Bhattacharyya and Mitesh M. Khapra and Pratyush Kumar},\n    year={2020},\n    booktitle={Findings of EMNLP},\n}\n"

lowerCamelCase :      Optional[Any]					 =     "\\n    IndicGLUE is a natural language understanding benchmark for Indian languages. It contains a wide\n    variety of tasks and covers 11 major Indian languages - as, bn, gu, hi, kn, ml, mr, or, pa, ta, te.\n"

lowerCamelCase :      int					 =     "\nCompute IndicGLUE evaluation metric associated to each IndicGLUE dataset.\nArgs:\n    predictions: list of predictions to score (as int64),\n        except for 'cvit-mkb-clsr' where each prediction is a vector (of float32).\n    references: list of ground truth labels corresponding to the predictions (as int64),\n        except for 'cvit-mkb-clsr' where each reference is a vector (of float32).\nReturns: depending on the IndicGLUE subset, one or several of:\n    \"accuracy\": Accuracy\n    \"f1\": F1 score\n    \"precision\": Precision@10\nExamples:\n\n    >>> indic_glue_metric = datasets.load_metric('indic_glue', 'wnli')  # 'wnli' or any of [\"copa\", \"sna\", \"csqa\", \"wstp\", \"inltkh\", \"bbca\", \"iitp-mr\", \"iitp-pr\", \"actsa-sc\", \"md\"]\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0}\n\n    >>> indic_glue_metric = datasets.load_metric('indic_glue', 'wiki-ner')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0, 'f1': 1.0}\n\n    >>> indic_glue_metric = datasets.load_metric('indic_glue', 'cvit-mkb-clsr')\n    >>> references = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]]\n    >>> predictions = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'precision@10': 1.0}\n\n"


def        _lowerCAmelCase					(						_UpperCamelCase						:     List[str]							, _UpperCamelCase						:     Tuple				)      ->			List[Any]:
     """simple docstring"""




     return float((preds == labels).mean()				)


def        _lowerCAmelCase					(						_UpperCamelCase						:     Union[str, Any]							, _UpperCamelCase						:     Any				)      ->			List[str]:
     """simple docstring"""




     _SCREAMING_SNAKE_CASE							    =simple_accuracy(_UpperCamelCase							, _UpperCamelCase				)
     _SCREAMING_SNAKE_CASE							    =float(fa_score(y_true=_UpperCamelCase							, y_pred=_UpperCamelCase				)				)
     return {
         "accuracy": acc,
         "f1": fa,
     }


def        _lowerCAmelCase					(						_UpperCamelCase						:     Any							, _UpperCamelCase						:     int				)      ->			str:
     """simple docstring"""




     _SCREAMING_SNAKE_CASE							    =np.array(_UpperCamelCase				)
     _SCREAMING_SNAKE_CASE							    =np.array(_UpperCamelCase				)
     _SCREAMING_SNAKE_CASE							    =en_sentvecs.shape[0]

     # mean centering
     _SCREAMING_SNAKE_CASE							    =en_sentvecs - np.mean(_UpperCamelCase							, axis=0				)
     _SCREAMING_SNAKE_CASE							    =in_sentvecs - np.mean(_UpperCamelCase							, axis=0				)

     _SCREAMING_SNAKE_CASE							    =cdist(_UpperCamelCase							, _UpperCamelCase							, 'cosine'				)
     _SCREAMING_SNAKE_CASE							    =np.array(range(_UpperCamelCase				)				)
     _SCREAMING_SNAKE_CASE							    =sim.argsort(axis=1				)[:, :10]
     _SCREAMING_SNAKE_CASE							    =np.any(preds == actual[:, None]							, axis=1				)
     return float(matches.mean()				)







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION     ,   _KWARGS_DESCRIPTION  )
class 							A__    (       datasets.Metric  ):





  def 						A  (		self      :	Any )	->    List[str]:





       '''simple docstring'''






       if self.config_name not in [
           "wnli",
           "copa",
           "sna",
           "csqa",
           "wstp",
           "inltkh",
           "bbca",
           "cvit-mkb-clsr",
           "iitp-mr",
           "iitp-pr",
           "actsa-sc",
           "md",
           "wiki-ner",
       ]:
            raise KeyError(
                'You should supply a configuration name selected in '
                '["wnli", "copa", "sna", "csqa", "wstp", "inltkh", "bbca", '
                '"cvit-mkb-clsr", "iitp-mr", "iitp-pr", "actsa-sc", "md", '
                '"wiki-ner"]' )
       return datasets.MetricInfo(
           description=_DESCRIPTION							,					citation=_CITATION							,					inputs_description=_KWARGS_DESCRIPTION							,					features=datasets.Features(
               {
                   'predictions': datasets.Value('int64' )
                   if self.config_name != 'cvit-mkb-clsr'
                   else datasets.Sequence(datasets.Value('float32' ) ),
                   'references': datasets.Value('int64' )
                   if self.config_name != 'cvit-mkb-clsr'
                   else datasets.Sequence(datasets.Value('float32' ) ),
               } )							,					codebase_urls=[]							,					reference_urls=[]							,					format='numpy' if self.config_name != 'cvit-mkb-clsr' else None							,					)





  def 						A  (		self      :	List[str]							,					_a      :	Tuple							,					_a      :	Optional[int] )	->    int:





       '''simple docstring'''






       if self.config_name == "cvit-mkb-clsr":
            return {"precision@10": precision_at_aa(_a							,					_a )}
       elif self.config_name in ["wiki-ner"]:
            return acc_and_fa(_a							,					_a )
       elif self.config_name in [
           "wnli",
           "copa",
           "sna",
           "csqa",
           "wstp",
           "inltkh",
           "bbca",
           "iitp-mr",
           "iitp-pr",
           "actsa-sc",
           "md",
       ]:
            return {"accuracy": simple_accuracy(_a							,					_a )}
       else:
            raise KeyError(
                'You should supply a configuration name selected in '
                '["wnli", "copa", "sna", "csqa", "wstp", "inltkh", "bbca", '
                '"cvit-mkb-clsr", "iitp-mr", "iitp-pr", "actsa-sc", "md", '
                '"wiki-ner"]' )
