



'''simple docstring'''



import importlib
from typing import List, Optional, Union

from seqeval.metrics import accuracy_score, classification_report

import datasets


_lowercase       :						List[str]						="\\n@inproceedings{ramshaw-marcus-1995-text,\n    title = \"Text Chunking using Transformation-Based Learning\",\n    author = \"Ramshaw, Lance  and\n      Marcus, Mitch\",\n    booktitle = \"Third Workshop on Very Large Corpora\",\n    year = \"1995\",\n    url = \"https://www.aclweb.org/anthology/W95-0107\",\n}\n@misc{seqeval,\n  title={{seqeval}: A Python framework for sequence labeling evaluation},\n  url={https://github.com/chakki-works/seqeval},\n  note={Software available from https://github.com/chakki-works/seqeval},\n  author={Hiroki Nakayama},\n  year={2018},\n}\n"

_lowercase       :						List[str]						="\\nseqeval is a Python framework for sequence labeling evaluation.\nseqeval can evaluate the performance of chunking tasks such as named-entity recognition, part-of-speech tagging, semantic role labeling and so on.\n\nThis is well-tested by using the Perl script conlleval, which can be used for\nmeasuring the performance of a system that has processed the CoNLL-2000 shared task data.\n\nseqeval supports following formats:\nIOB1\nIOB2\nIOE1\nIOE2\nIOBES\n\nSee the [README.md] file at https://github.com/chakki-works/seqeval for more information.\n"

_lowercase       :						int						="\nProduces labelling scores along with its sufficient statistics\nfrom a source against one or more references.\n\nArgs:\n    predictions: List of List of predicted labels (Estimated targets as returned by a tagger)\n    references: List of List of reference labels (Ground truth (correct) target values)\n    suffix: True if the IOB prefix is after type, False otherwise. default: False\n    scheme: Specify target tagging scheme. Should be one of [\"IOB1\", \"IOB2\", \"IOE1\", \"IOE2\", \"IOBES\", \"BILOU\"].\n        default: None\n    mode: Whether to count correct entity labels with incorrect I/B tags as true positives or not.\n        If you want to only count exact matches, pass mode=\"strict\". default: None.\n    sample_weight: Array-like of shape (n_samples,), weights for individual samples. default: None\n    zero_division: Which value to substitute as a metric value when encountering zero division. Should be on of 0, 1,\n        \"warn\". \"warn\" acts as 0, but the warning is raised.\n\nReturns:\n    'scores': dict. Summary of the scores for overall and per type\n        Overall:\n            'accuracy': accuracy,\n            'precision': precision,\n            'recall': recall,\n            'f1': F1 score, also known as balanced F-score or F-measure,\n        Per type:\n            'precision': precision,\n            'recall': recall,\n            'f1': F1 score, also known as balanced F-score or F-measure\nExamples:\n\n    >>> predictions = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']]\n    >>> references = [['O', 'O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']]\n    >>> seqeval = datasets.load_metric(\"seqeval\")\n    >>> results = seqeval.compute(predictions=predictions, references=references)\n    >>> print(list(results.keys()))\n    ['MISC', 'PER', 'overall_precision', 'overall_recall', 'overall_f1', 'overall_accuracy']\n    >>> print(results[\"overall_f1\"])\n    0.5\n    >>> print(results[\"PER\"][\"f1\"])\n    1.0\n"






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION,	_KWARGS_DESCRIPTION      )
class 			_SCREAMING_SNAKE_CASE				(datasets.Metric      ):



			def 				lowerCAmelCase (     self		:						Union[str, Any]			)      ->      Dict:
							"""simple docstring"""





							return datasets.MetricInfo(
							    description=_DESCRIPTION    ,			citation=_CITATION    ,			homepage='''https://github.com/chakki-works/seqeval'''    ,			inputs_description=_KWARGS_DESCRIPTION    ,			features=datasets.Features(
							        {
							            '''predictions''': datasets.Sequence(datasets.Value('''string'''    ,			id='''label'''			)    ,			id='''sequence'''			),
							            '''references''': datasets.Sequence(datasets.Value('''string'''    ,			id='''label'''			)    ,			id='''sequence'''			),
							        }			)    ,			codebase_urls=['''https://github.com/chakki-works/seqeval''']    ,			reference_urls=['''https://github.com/chakki-works/seqeval''']    ,			)







			def 				lowerCAmelCase (     self		:						Any    ,			__UpperCamelCase		:						Dict    ,			__UpperCamelCase		:						int    ,			__UpperCamelCase		:						bool = False    ,			__UpperCamelCase		:						Optional[str] = None    ,			__UpperCamelCase		:						Optional[str] = None    ,			__UpperCamelCase		:						Optional[List[int]] = None    ,			__UpperCamelCase		:						Union[str, int] = "warn"    ,			)      ->      Optional[Any]:
							"""simple docstring"""





							if scheme is not None:
											try:
															snake_case__      :   Any      =    importlib.import_module('''seqeval.scheme'''			)
															snake_case__      :   int      =    getattr(__UpperCamelCase    ,			__UpperCamelCase			)
											except AttributeError:
															raise ValueError(F'''Scheme should be one of [IOB1, IOB2, IOE1, IOE2, IOBES, BILOU], got {scheme}'''			)
							snake_case__      :   int      =    classification_report(
							    y_true=__UpperCamelCase    ,			y_pred=__UpperCamelCase    ,			suffix=__UpperCamelCase    ,			output_dict=__UpperCamelCase    ,			scheme=__UpperCamelCase    ,			mode=__UpperCamelCase    ,			sample_weight=__UpperCamelCase    ,			zero_division=__UpperCamelCase    ,			)
							report.pop('''macro avg'''			)
							report.pop('''weighted avg'''			)
							snake_case__      :   List[Any]      =    report.pop('''micro avg'''			)

							snake_case__      :   str      =    {
							    type_name: {
							        '''precision''': score['''precision'''],
							        '''recall''': score['''recall'''],
							        '''f1''': score['''f1-score'''],
							        '''number''': score['''support'''],
							    }
							    for type_name, score in report.items()
							}
							snake_case__      :   Tuple      =    overall_score['''precision''']
							snake_case__      :   Dict      =    overall_score['''recall''']
							snake_case__      :   Dict      =    overall_score['''f1-score''']
							snake_case__      :   Optional[int]      =    accuracy_score(y_true=__UpperCamelCase    ,			y_pred=__UpperCamelCase			)

							return scores












'''simple docstring'''


import json
import os
import shutil
import tempfile
import unittest

from transformers import is_speech_available, is_torch_available
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_torch
from transformers.utils import FEATURE_EXTRACTOR_NAME


if is_speech_available() and is_torch_available():
					from transformers import SpeechTaFeatureExtractor, SpeechTaProcessor

					from .test_feature_extraction_speechta import floats_list


lowerCamelCase_					:			List[str]     						=				get_tests_dir('''fixtures/test_sentencepiece_bpe_char.model''')
@require_torch
class  _SCREAMING_SNAKE_CASE				(		unittest.TestCase						):


			'''simple docstring'''





			def 					A   ( self				:    Union[str, Any]      )				->     Union[str, Any]:

									'''simple docstring'''







									UpperCamelCase__       =			tempfile.mkdtemp()

									UpperCamelCase__       =			SpeechTaTokenizer(lowercase      )
									tokenizer.save_pretrained(self.tmpdirname      )

									UpperCamelCase__       =			{
									    """feature_size""": 1,
									    """padding_value""": 0.0,
									    """sampling_rate""": 1_6_0_0_0,
									    """do_normalize""": False,
									    """num_mel_bins""": 8_0,
									    """hop_length""": 1_6,
									    """win_length""": 6_4,
									    """win_function""": """hann_window""",
									    """fmin""": 8_0,
									    """fmax""": 7_6_0_0,
									    """mel_floor""": 1e-10,
									    """reduction_factor""": 2,
									    """return_attention_mask""": True,
									}

									UpperCamelCase__       =			os.path.join(self.tmpdirname			,   lowercase      )
									with open(self.feature_extraction_file			,   """w"""			,   encoding="""utf-8"""      ) as fp:
															fp.write(json.dumps(lowercase      ) + """\n"""      )






			def 					A   ( self				:    Union[str, Any]			,   **lowercase				:    List[Any]      )				->     Any:

									'''simple docstring'''







									return SpeechTaTokenizer.from_pretrained(self.tmpdirname			,   **lowercase      )






			def 					A   ( self				:    str			,   **lowercase				:    Tuple      )				->     Optional[Any]:

									'''simple docstring'''







									return SpeechTaFeatureExtractor.from_pretrained(self.tmpdirname			,   **lowercase      )






			def 					A   ( self				:    Union[str, Any]      )				->     str:

									'''simple docstring'''







									shutil.rmtree(self.tmpdirname      )






			def 					A   ( self				:    List[Any]      )				->     Optional[int]:

									'''simple docstring'''







									UpperCamelCase__       =			self.get_tokenizer()
									UpperCamelCase__       =			self.get_feature_extractor()

									UpperCamelCase__       =			SpeechTaProcessor(tokenizer=lowercase			,   feature_extractor=lowercase      )

									processor.save_pretrained(self.tmpdirname      )
									UpperCamelCase__       =			SpeechTaProcessor.from_pretrained(self.tmpdirname      )

									self.assertEqual(processor.tokenizer.get_vocab()			,   tokenizer.get_vocab()      )
									self.assertIsInstance(processor.tokenizer			,   lowercase      )

									self.assertEqual(processor.feature_extractor.to_json_string()			,   feature_extractor.to_json_string()      )
									self.assertIsInstance(processor.feature_extractor			,   lowercase      )






			def 					A   ( self				:    Tuple      )				->     Union[str, Any]:

									'''simple docstring'''







									UpperCamelCase__       =			SpeechTaProcessor(tokenizer=self.get_tokenizer()			,   feature_extractor=self.get_feature_extractor()      )
									processor.save_pretrained(self.tmpdirname      )

									UpperCamelCase__       =			self.get_tokenizer(bos_token="""(BOS)"""			,   eos_token="""(EOS)"""      )
									UpperCamelCase__       =			self.get_feature_extractor(do_normalize=lowercase			,   padding_value=1.0      )

									UpperCamelCase__       =			SpeechTaProcessor.from_pretrained(
									    self.tmpdirname			,   bos_token="""(BOS)"""			,   eos_token="""(EOS)"""			,   do_normalize=lowercase			,   padding_value=1.0      )

									self.assertEqual(processor.tokenizer.get_vocab()			,   tokenizer_add_kwargs.get_vocab()      )
									self.assertIsInstance(processor.tokenizer			,   lowercase      )

									self.assertEqual(processor.feature_extractor.to_json_string()			,   feature_extractor_add_kwargs.to_json_string()      )
									self.assertIsInstance(processor.feature_extractor			,   lowercase      )






			def 					A   ( self				:    List[Any]      )				->     Dict:

									'''simple docstring'''







									UpperCamelCase__       =			self.get_feature_extractor()
									UpperCamelCase__       =			self.get_tokenizer()

									UpperCamelCase__       =			SpeechTaProcessor(tokenizer=lowercase			,   feature_extractor=lowercase      )

									UpperCamelCase__       =			floats_list((3, 1_0_0_0)      )

									UpperCamelCase__       =			feature_extractor(audio=lowercase			,   return_tensors="""np"""      )
									UpperCamelCase__       =			processor(audio=lowercase			,   return_tensors="""np"""      )

									for key in input_feat_extract.keys():
															self.assertAlmostEqual(input_feat_extract[key].sum()			,   input_processor[key].sum()			,   delta=1e-2      )






			def 					A   ( self				:    List[str]      )				->     Optional[Any]:

									'''simple docstring'''







									UpperCamelCase__       =			self.get_feature_extractor()
									UpperCamelCase__       =			self.get_tokenizer()

									UpperCamelCase__       =			SpeechTaProcessor(tokenizer=lowercase			,   feature_extractor=lowercase      )

									UpperCamelCase__       =			floats_list((3, 1_0_0_0)      )

									UpperCamelCase__       =			feature_extractor(audio_target=lowercase			,   return_tensors="""np"""      )
									UpperCamelCase__       =			processor(audio_target=lowercase			,   return_tensors="""np"""      )

									for key in input_feat_extract.keys():
															self.assertAlmostEqual(input_feat_extract[key].sum()			,   input_processor[key].sum()			,   delta=1e-2      )






			def 					A   ( self				:    Optional[Any]      )				->     Optional[Any]:

									'''simple docstring'''







									UpperCamelCase__       =			self.get_feature_extractor()
									UpperCamelCase__       =			self.get_tokenizer()

									UpperCamelCase__       =			SpeechTaProcessor(tokenizer=lowercase			,   feature_extractor=lowercase      )

									UpperCamelCase__       =			"""This is a test string"""

									UpperCamelCase__       =			processor(text=lowercase      )
									UpperCamelCase__       =			tokenizer(lowercase      )

									for key in encoded_tok.keys():
															self.assertListEqual(encoded_tok[key]			,   encoded_processor[key]      )






			def 					A   ( self				:    List[Any]      )				->     int:

									'''simple docstring'''







									UpperCamelCase__       =			self.get_feature_extractor()
									UpperCamelCase__       =			self.get_tokenizer()

									UpperCamelCase__       =			SpeechTaProcessor(tokenizer=lowercase			,   feature_extractor=lowercase      )

									UpperCamelCase__       =			"""This is a test string"""

									UpperCamelCase__       =			processor(text_target=lowercase      )
									UpperCamelCase__       =			tokenizer(lowercase      )

									for key in encoded_tok.keys():
															self.assertListEqual(encoded_tok[key]			,   encoded_processor[key]      )






			def 					A   ( self				:    Any      )				->     Tuple:

									'''simple docstring'''







									UpperCamelCase__       =			self.get_feature_extractor()
									UpperCamelCase__       =			self.get_tokenizer()

									UpperCamelCase__       =			SpeechTaProcessor(tokenizer=lowercase			,   feature_extractor=lowercase      )

									UpperCamelCase__       =			[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

									UpperCamelCase__       =			processor.batch_decode(lowercase      )
									UpperCamelCase__       =			tokenizer.batch_decode(lowercase      )

									self.assertListEqual(lowercase			,   lowercase      )







			def 					A   ( self				:    List[Any]      )				->     List[str]:

									'''simple docstring'''







									UpperCamelCase__       =			self.get_feature_extractor()
									UpperCamelCase__       =			self.get_tokenizer()

									UpperCamelCase__       =			SpeechTaProcessor(tokenizer=lowercase			,   feature_extractor=lowercase      )

									self.assertListEqual(
									    processor.model_input_names			,   feature_extractor.model_input_names			,   msg="""`processor` and `feature_extractor` model input names do not match"""			,   )
