

import itertools
import os
import random
import tempfile
import unittest

import numpy as np

from transformers import TvltFeatureExtractor, is_datasets_available
from transformers.testing_utils import check_json_file_has_correct_format, require_torch, require_torchaudio
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_torch_available():
			import torch

if is_datasets_available():
			from datasets import load_dataset

a_     =			random.Random()

def        lowerCamelCase__       (    _a							,	_a=1.0							,	_a=None							,	_a=None):
	if rng is None:
		SCREAMING_SNAKE_CASE      :					int    =				global_rng

	SCREAMING_SNAKE_CASE      :					Tuple    =				[]
	for batch_idx in range(shape[0]):
		values.append([])
		for _ in range(shape[1]):
			values[-1].append(rng.random() * scale)

	return values






class _UpperCamelCase							(							unittest.TestCase      ):
			'''simple docstring'''







			def __init__(	self							:       List[str]					,					a							:       Any					,					a							:       Dict=7					,					a							:       int=400					,					a							:       Tuple=2000					,					a							:       Union[str, Any]=2048					,					a							:       Dict=128					,					a							:       Union[str, Any]=1					,					a							:       List[Any]=512					,					a							:       Any=30					,					a							:       int=4_4100					,					)							->				Optional[Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					Any    =				parent
				SCREAMING_SNAKE_CASE      :					int    =				batch_size
				SCREAMING_SNAKE_CASE      :					Optional[int]    =				min_seq_length
				SCREAMING_SNAKE_CASE      :					Optional[int]    =				max_seq_length
				SCREAMING_SNAKE_CASE      :					List[Any]    =				(self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
				SCREAMING_SNAKE_CASE      :					Optional[int]    =				spectrogram_length
				SCREAMING_SNAKE_CASE      :					List[Any]    =				feature_size
				SCREAMING_SNAKE_CASE      :					int    =				num_audio_channels
				SCREAMING_SNAKE_CASE      :					Union[str, Any]    =				hop_length
				SCREAMING_SNAKE_CASE      :					Optional[int]    =				chunk_length
				SCREAMING_SNAKE_CASE      :					str    =				sampling_rate
			def 							__UpperCamelCase       (	self							:       Optional[Any]		)							->				Any:

				"""simple docstring"""

				return {
				    "spectrogram_length": self.spectrogram_length,
				    "feature_size": self.feature_size,
				    "num_audio_channels": self.num_audio_channels,
				    "hop_length": self.hop_length,
				    "chunk_length": self.chunk_length,
				    "sampling_rate": self.sampling_rate,
				}






			def 							__UpperCamelCase       (	self							:       List[Any]					,					a							:       List[str]=False					,					a							:       Dict=False		)							->				Any:

				"""simple docstring"""

				def _flatten(a							:       Optional[Any]		):
					return list(itertools.chain(*a		)		)

				if equal_length:
					SCREAMING_SNAKE_CASE      :					Dict    =				[floats_list((self.max_seq_length, self.feature_size)		) for _ in range(self.batch_size		)]
				else:
					# make sure that inputs increase in size
					SCREAMING_SNAKE_CASE      :					Any    =				[
					    floats_list((x, self.feature_size)		)
					    for x in range(self.min_seq_length					,					self.max_seq_length					,					self.seq_length_diff		)
					]
				if numpify:
					SCREAMING_SNAKE_CASE      :					Any    =				[np.asarray(a		) for x in speech_inputs]
				return speech_inputs






@require_torch
@require_torchaudio
class _UpperCamelCase							(							__A							,		unittest.TestCase      ):
			'''simple docstring'''







			lowerCamelCase__							=TvltFeatureExtractor
			def 							__UpperCamelCase       (	self							:       int		)							->				Optional[int]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					int    =				TvltFeatureExtractionTester(self		)
			def 							__UpperCamelCase       (	self							:       Any		)							->				Optional[int]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					Tuple    =				self.feature_extraction_class(**self.feat_extract_dict		)
				self.assertTrue(hasattr(a					,					"spectrogram_length"		)		)
				self.assertTrue(hasattr(a					,					"feature_size"		)		)
				self.assertTrue(hasattr(a					,					"num_audio_channels"		)		)
				self.assertTrue(hasattr(a					,					"hop_length"		)		)
				self.assertTrue(hasattr(a					,					"chunk_length"		)		)
				self.assertTrue(hasattr(a					,					"sampling_rate"		)		)
			def 							__UpperCamelCase       (	self							:       Optional[Any]		)							->				int:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					str    =				self.feature_extraction_class(**self.feat_extract_dict		)

				with tempfile.TemporaryDirectory() as tmpdirname:
					SCREAMING_SNAKE_CASE      :					Tuple    =				feat_extract_first.save_pretrained(a		)[0]
					check_json_file_has_correct_format(a		)
					SCREAMING_SNAKE_CASE      :					List[str]    =				self.feature_extraction_class.from_pretrained(a		)

				SCREAMING_SNAKE_CASE      :					str    =				feat_extract_first.to_dict()
				SCREAMING_SNAKE_CASE      :					List[str]    =				feat_extract_second.to_dict()
				SCREAMING_SNAKE_CASE      :					Tuple    =				dict_first.pop("mel_filters"		)
				SCREAMING_SNAKE_CASE      :					List[Any]    =				dict_second.pop("mel_filters"		)
				self.assertTrue(np.allclose(a					,					a		)		)
				self.assertEqual(a					,					a		)
			def 							__UpperCamelCase       (	self							:       int		)							->				List[str]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					Union[str, Any]    =				self.feature_extraction_class(**self.feat_extract_dict		)

				with tempfile.TemporaryDirectory() as tmpdirname:
					SCREAMING_SNAKE_CASE      :					Optional[Any]    =				os.path.join(a					,					"feat_extract.json"		)
					feat_extract_first.to_json_file(a		)
					SCREAMING_SNAKE_CASE      :					List[Any]    =				self.feature_extraction_class.from_json_file(a		)

				SCREAMING_SNAKE_CASE      :					int    =				feat_extract_first.to_dict()
				SCREAMING_SNAKE_CASE      :					Tuple    =				feat_extract_second.to_dict()
				SCREAMING_SNAKE_CASE      :					Dict    =				dict_first.pop("mel_filters"		)
				SCREAMING_SNAKE_CASE      :					Any    =				dict_second.pop("mel_filters"		)
				self.assertTrue(np.allclose(a					,					a		)		)
				self.assertEqual(a					,					a		)
			def 							__UpperCamelCase       (	self							:       int		)							->				Any:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					List[str]    =				self.feature_extraction_class(**self.feat_extract_dict		)

				# create three inputs of length 800, 1000, and 1200
				SCREAMING_SNAKE_CASE      :					Dict    =				[floats_list((1, x)		)[0] for x in range(800					,					1400					,					200		)]
				SCREAMING_SNAKE_CASE      :					List[Any]    =				[np.asarray(a		) for speech_input in speech_inputs]

				# Test not batched input
				SCREAMING_SNAKE_CASE      :					int    =				feature_extractor(np_speech_inputs[0]					,					return_tensors="np"					,					sampling_rate=4_4100		).audio_values

				self.assertTrue(encoded_audios.ndim == 4		)
				self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size		)
				self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length		)
				self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels		)

				# Test batched
				SCREAMING_SNAKE_CASE      :					List[str]    =				feature_extractor(a					,					return_tensors="np"					,					sampling_rate=4_4100		).audio_values

				self.assertTrue(encoded_audios.ndim == 4		)
				self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size		)
				self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length		)
				self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels		)

				# Test audio masking
				SCREAMING_SNAKE_CASE      :					Optional[int]    =				feature_extractor(
				    a					,					return_tensors="np"					,					sampling_rate=4_4100					,					mask_audio=a		).audio_values

				self.assertTrue(encoded_audios.ndim == 4		)
				self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size		)
				self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length		)
				self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels		)

				# Test 2-D numpy arrays are batched.
				SCREAMING_SNAKE_CASE      :					int    =				[floats_list((1, x)		)[0] for x in (800, 800, 800)]
				SCREAMING_SNAKE_CASE      :					Optional[int]    =				np.asarray(a		)
				SCREAMING_SNAKE_CASE      :					Union[str, Any]    =				feature_extractor(a					,					return_tensors="np"					,					sampling_rate=4_4100		).audio_values
				self.assertTrue(encoded_audios.ndim == 4		)
				self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size		)
				self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length		)
				self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels		)
			def 							__UpperCamelCase       (	self							:       Tuple					,					a							:       str		)							->				int:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					str    =				load_dataset("hf-internal-testing/librispeech_asr_dummy"					,					"clean"					,					split="validation"		)
				# automatic decoding with librispeech
				SCREAMING_SNAKE_CASE      :					List[str]    =				ds.sort("id"		).select(range(a		)		)[:num_samples]["audio"]

				return [x["array"] for x in speech_samples]






			def 							__UpperCamelCase       (	self							:       Dict		)							->				Union[str, Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					str    =				self._load_datasamples(1		)
				SCREAMING_SNAKE_CASE      :					Optional[Any]    =				TvltFeatureExtractor()
				SCREAMING_SNAKE_CASE      :					Optional[int]    =				feature_extractor(a					,					return_tensors="pt"		).audio_values

				self.assertEquals(audio_values.shape					,					(1, 1, 192, 128)		)

				SCREAMING_SNAKE_CASE      :					List[str]    =				torch.tensor([[-0.3032, -0.2708], [-0.4434, -0.4007]]		)
				self.assertTrue(torch.allclose(audio_values[0, 0, :2, :2]					,					a					,					atol=1e-4		)		)




'''simple docstring'''





from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


_lowerCAmelCase      						=	'''\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
'''

_lowerCAmelCase      						=	'''\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
'''

_lowerCAmelCase      						=	'''
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for \'record\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'prediction_text\': the predicted answer text
        - for \'multirc\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question-answer pair as specified by the dataset
            - \'prediction\': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for \'record\': list of question-answers dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'answers\': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for \'record\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1\': F1 score
    - for \'multirc\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1_m\': Per-question macro-F1 score
        - \'f1_a\': Average F1 score over all answers
    - for \'axb\':
        \'matthews_correlation\': Matthew Correlation
    - for \'cb\':
        - \'accuracy\': Accuracy
        - \'f1\': F1 score
    - for all others:
        - \'accuracy\': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')
    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]
    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')
    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'matthews_correlation\': 1.0}
'''
def    _SCREAMING_SNAKE_CASE  (       UpperCamelCase							,    UpperCamelCase			):



    """simple docstring"""





    return float((preds == labels).mean()			)
def    _SCREAMING_SNAKE_CASE  (       UpperCamelCase							,    UpperCamelCase							,    UpperCamelCase="binary"			):



    """simple docstring"""





    lowerCAmelCase__ :  Any							      =					simple_accuracy(UpperCamelCase							,    UpperCamelCase			)
    lowerCAmelCase__ :  Tuple							      =					float(fa_score(y_true=UpperCamelCase							,    y_pred=UpperCamelCase							,    average=UpperCamelCase			)			)
    return {
        "accuracy": acc,
        "f1": fa,
    }
def    _SCREAMING_SNAKE_CASE  (       UpperCamelCase							,    UpperCamelCase			):



    """simple docstring"""





    lowerCAmelCase__ :  List[str]							      =					{}
    for id_pred, label in zip(UpperCamelCase							,    UpperCamelCase			):
        lowerCAmelCase__ :  str							      =					f"""{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}"""
        lowerCAmelCase__ :  Dict							      =					id_pred["""prediction"""]
        if question_id in question_map:
            question_map[question_id].append((pred, label)			)
        else:
            lowerCAmelCase__ :  Optional[int]							      =					[(pred, label)]
    lowerCAmelCase__      ,   lowerCAmelCase__ :  int							      =					[], []
    for question, preds_labels in question_map.items():
        lowerCAmelCase__      ,   lowerCAmelCase__ :  Optional[int]							      =					zip(*UpperCamelCase			)
        lowerCAmelCase__ :  List[Any]							      =					fa_score(y_true=UpperCamelCase							,    y_pred=UpperCamelCase							,    average="""macro"""			)
        fas.append(UpperCamelCase			)
        lowerCAmelCase__ :  Union[str, Any]							      =					int(sum(pred == label for pred, label in preds_labels			) == len(UpperCamelCase			)			)
        ems.append(UpperCamelCase			)
    lowerCAmelCase__ :  Optional[Any]							      =					float(sum(UpperCamelCase			) / len(UpperCamelCase			)			)
    lowerCAmelCase__ :  List[Any]							      =					sum(UpperCamelCase			) / len(UpperCamelCase			)
    lowerCAmelCase__ :  Dict							      =					float(fa_score(y_true=UpperCamelCase							,    y_pred=[id_pred["""prediction"""] for id_pred in ids_preds]			)			)
    return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION						, _KWARGS_DESCRIPTION     )
class        lowerCAmelCase_(				datasets.Metric     ):
  '''simple docstring'''




  def 							UpperCAmelCase_	(    self      )    ->		Optional[Any]:
      if self.config_name not in [
          "boolq",
          "cb",
          "copa",
          "multirc",
          "record",
          "rte",
          "wic",
          "wsc",
          "wsc.fixed",
          "axb",
          "axg",
      ]:
          raise KeyError(
              """You should supply a configuration name selected in """
              """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"""      )
      return datasets.MetricInfo(
          description=_DESCRIPTION  ,citation=_CITATION  ,inputs_description=_KWARGS_DESCRIPTION  ,features=datasets.Features(self._get_feature_types()      )  ,codebase_urls=[]  ,reference_urls=[]  ,format="""numpy""" if not self.config_name == """record""" and not self.config_name == """multirc""" else None  ,)




  def 							UpperCAmelCase_	(    self      )    ->		str:
      if self.config_name == "record":
          return {
              "predictions": {
                  "idx": {
                      "passage": datasets.Value("""int64"""      ),
                      "query": datasets.Value("""int64"""      ),
                  },
                  "prediction_text": datasets.Value("""string"""      ),
              },
              "references": {
                  "idx": {
                      "passage": datasets.Value("""int64"""      ),
                      "query": datasets.Value("""int64"""      ),
                  },
                  "answers": datasets.Sequence(datasets.Value("""string"""      )      ),
              },
          }
      elif self.config_name == "multirc":
          return {
              "predictions": {
                  "idx": {
                      "answer": datasets.Value("""int64"""      ),
                      "paragraph": datasets.Value("""int64"""      ),
                      "question": datasets.Value("""int64"""      ),
                  },
                  "prediction": datasets.Value("""int64"""      ),
              },
              "references": datasets.Value("""int64"""      ),
          }
      else:
          return {
              "predictions": datasets.Value("""int64"""      ),
              "references": datasets.Value("""int64"""      ),
          }






  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase      )    ->		Any:
      if self.config_name == "axb":
          return {"matthews_correlation": matthews_corrcoef(__UpperCAmelCase  ,__UpperCAmelCase      )}
      elif self.config_name == "cb":
          return acc_and_fa(__UpperCAmelCase  ,__UpperCAmelCase  ,fa_avg="""macro"""      )
      elif self.config_name == "record":
          lowerCAmelCase__ :  Optional[Any]							      =					[
              {
                  """qas""": [
                      {"""id""": ref["""idx"""]["""query"""], """answers""": [{"""text""": ans} for ans in ref["""answers"""]]}
                      for ref in references
                  ]
              }
          ]
          lowerCAmelCase__ :  Union[str, Any]							      =					{pred["""idx"""]["""query"""]: pred["""prediction_text"""] for pred in predictions}
          return evaluate_record(__UpperCAmelCase  ,__UpperCAmelCase      )[0]
      elif self.config_name == "multirc":
          return evaluate_multirc(__UpperCAmelCase  ,__UpperCAmelCase      )
      elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
          return {"accuracy": simple_accuracy(__UpperCAmelCase  ,__UpperCAmelCase      )}
      else:
          raise KeyError(
              """You should supply a configuration name selected in """
              """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"""      )


