




"""simple docstring"""




import datasets

from .evaluate import evaluate


lowerCAmelCase							:			Tuple     							=							"""\
@article{hendrycks2021cuad,
      title={CUAD: An Expert-Annotated NLP Dataset for Legal Contract Review},
      author={Dan Hendrycks and Collin Burns and Anya Chen and Spencer Ball},
      journal={arXiv preprint arXiv:2103.06268},
      year={2021}
}
"""

lowerCAmelCase							:			str     							=							"""
This metric wrap the official scoring script for version 1 of the Contract
Understanding Atticus Dataset (CUAD).
Contract Understanding Atticus Dataset (CUAD) v1 is a corpus of more than 13,000 labels in 510
commercial legal contracts that have been manually labeled to identify 41 categories of important
clauses that lawyers look for when reviewing contracts in connection with corporate transactions.
"""

lowerCAmelCase							:			Optional[int]     							=							"""
Computes CUAD scores (EM, F1, AUPR, Precision@80%Recall, and Precision@90%Recall).
Args:
    predictions: List of question-answers dictionaries with the following key-values:
        - 'id': id of the question-answer pair as given in the references (see below)
        - 'prediction_text': list of possible texts for the answer, as a list of strings
        depending on a threshold on the confidence probability of each prediction.
    references: List of question-answers dictionaries with the following key-values:
        - 'id': id of the question-answer pair (see above),
        - 'answers': a Dict in the CUAD dataset format
            {
                'text': list of possible texts for the answer, as a list of strings
                'answer_start': list of start positions for the answer, as a list of ints
            }
            Note that answer_start values are not taken into account to compute the metric.
Returns:
    'exact_match': Exact match (the normalized answer exactly match the gold answer)
    'f1': The F-score of predicted tokens versus the gold answer
    'aupr': Area Under the Precision-Recall curve
    'prec_at_80_recall': Precision at 80% recall
    'prec_at_90_recall': Precision at 90% recall
Examples:
    >>> predictions = [{'prediction_text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}]
    >>> references = [{'answers': {'answer_start': [143, 49], 'text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.']}, 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}]
    >>> cuad_metric = datasets.load_metric(\"cuad\")
    >>> results = cuad_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 100.0, 'f1': 100.0, 'aupr': 0.0, 'prec_at_80_recall': 1.0, 'prec_at_90_recall': 1.0}
"""






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION						,					_KWARGS_DESCRIPTION				)
class     __magic_name__							(       datasets.Metric				):






				'''simple docstring'''





				def    _lowerCAmelCase				(       self ):





							"""simple docstring"""
							return datasets.MetricInfo(
							    description=_DESCRIPTION		, citation=_CITATION		, inputs_description=_KWARGS_DESCRIPTION		, features=datasets.Features(
							        {
							            """predictions""": {
							                """id""": datasets.Value("""string""" ),
							                """prediction_text""": datasets.features.Sequence(datasets.Value("""string""" ) ),
							            },
							            """references""": {
							                """id""": datasets.Value("""string""" ),
							                """answers""": datasets.features.Sequence(
							                    {
							                        """text""": datasets.Value("""string""" ),
							                        """answer_start""": datasets.Value("""int32""" ),
							                    } ),
							            },
							        } )		, codebase_urls=["""https://www.atticusprojectai.org/cuad"""]		, reference_urls=["""https://www.atticusprojectai.org/cuad"""]		, )





				def    _lowerCAmelCase				(       self		, _a		, _a ):





							"""simple docstring"""
							lowerCamelCase			      =					{prediction["id"]: prediction["prediction_text"] for prediction in predictions}
							lowerCamelCase			      =					[
							    {
							        "paragraphs": [
							            {
							                "qas": [
							                    {
							                        "answers": [{"text": answer_text} for answer_text in ref["answers"]["text"]],
							                        "id": ref["id"],
							                    }
							                    for ref in references
							                ]
							            }
							        ]
							    }
							]
							lowerCamelCase			      =					evaluate(dataset=_a		, predictions=_a )
							return score




'''simple docstring'''




import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


__SCREAMING_SNAKE_CASE       :     Union[str, Any]     =  get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
  from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

__SCREAMING_SNAKE_CASE       :     Optional[int]     =  256_047
__SCREAMING_SNAKE_CASE       :     Optional[int]     =  256_145

@require_sentencepiece
@require_tokenizers
class 	lowerCamelCase_	(snake_case__							, unittest.TestCase    ):
      '''simple docstring'''

      __UpperCamelCase:  int						=						NllbTokenizer
      __UpperCamelCase:  Tuple						=						NllbTokenizerFast
      __UpperCamelCase:  Union[str, Any]						=						True
      __UpperCamelCase:  Dict						=						True
      __UpperCamelCase:  Optional[Any]						=						{}




      def      _A	(       self						:				Union[str, Any]       ):
             super().setUp()

             # We have a SentencePiece fixture for testing
             _UpperCAmelCase     :  Tuple							      =    NllbTokenizer(A						,					keep_accents=A       )
             tokenizer.save_pretrained(self.tmpdirname       )




      def      _A	(       self						:				Dict       ):
             _UpperCAmelCase     :  Tuple							      =    NllbTokenizer(A						,					keep_accents=A       )

             _UpperCAmelCase     :  Optional[Any]							      =    tokenizer.tokenize("This is a test"       )
             self.assertListEqual(A						,					["▁This", "▁is", "▁a", "▁t", "est"]       )

             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(A       )						,					[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]						,					)

             _UpperCAmelCase     :  List[str]							      =    tokenizer.tokenize("I was born in 92000, and this is falsé."       )
             self.assertListEqual(
                 A						,					[
                     SPIECE_UNDERLINE + "I",
                     SPIECE_UNDERLINE + "was",
                     SPIECE_UNDERLINE + "b",
                     "or",
                     "n",
                     SPIECE_UNDERLINE + "in",
                     SPIECE_UNDERLINE + "",
                     "9",
                     "2",
                     "0",
                     "0",
                     "0",
                     ",",
                     SPIECE_UNDERLINE + "and",
                     SPIECE_UNDERLINE + "this",
                     SPIECE_UNDERLINE + "is",
                     SPIECE_UNDERLINE + "f",
                     "al",
                     "s",
                     "é",
                     ".",
                 ]						,					)
             _UpperCAmelCase     :  Optional[Any]							      =    tokenizer.convert_tokens_to_ids(A       )
             self.assertListEqual(
                 A						,					[
                     value + tokenizer.fairseq_offset
                     for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                 ]						,					)

             _UpperCAmelCase     :  Union[str, Any]							      =    tokenizer.convert_ids_to_tokens(A       )
             self.assertListEqual(
                 A						,					[
                     SPIECE_UNDERLINE + "I",
                     SPIECE_UNDERLINE + "was",
                     SPIECE_UNDERLINE + "b",
                     "or",
                     "n",
                     SPIECE_UNDERLINE + "in",
                     SPIECE_UNDERLINE + "",
                     "<unk>",
                     "2",
                     "0",
                     "0",
                     "0",
                     ",",
                     SPIECE_UNDERLINE + "and",
                     SPIECE_UNDERLINE + "this",
                     SPIECE_UNDERLINE + "is",
                     SPIECE_UNDERLINE + "f",
                     "al",
                     "s",
                     "<unk>",
                     ".",
                 ]						,					)




      def      _A	(       self						:				List[Any]       ):
             _UpperCAmelCase     :  Any							      =    (self.rust_tokenizer_class, "hf-internal-testing/tiny-random-nllb", {})
             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                    with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
                           _UpperCAmelCase     :  Dict							      =    self.rust_tokenizer_class.from_pretrained(A						,					**A       )
                           _UpperCAmelCase     :  str							      =    self.tokenizer_class.from_pretrained(A						,					**A       )

                           _UpperCAmelCase     :  Optional[int]							      =    tempfile.mkdtemp()

                           _UpperCAmelCase     :  Dict							      =    tokenizer_r.save_pretrained(A       )
                           _UpperCAmelCase     :  Dict							      =    tokenizer_p.save_pretrained(A       )

                           # Checks it save with the same files + the tokenizer.json file for the fast one
                           self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files       )       )
                           _UpperCAmelCase     :  Optional[int]							      =    tuple(f for f in tokenizer_r_files if "tokenizer.json" not in f       )
                           self.assertSequenceEqual(A						,					A       )

                           # Checks everything loads correctly in the same way
                           _UpperCAmelCase     :  List[Any]							      =    tokenizer_r.from_pretrained(A       )
                           _UpperCAmelCase     :  List[str]							      =    tokenizer_p.from_pretrained(A       )

                           # Check special tokens are set accordingly on Rust and Python
                           for key in tokenizer_pp.special_tokens_map:
                                  self.assertTrue(hasattr(A						,					A       )       )

                           shutil.rmtree(A       )

                           # Save tokenizer rust, legacy_format=True
                           _UpperCAmelCase     :  Optional[Any]							      =    tempfile.mkdtemp()

                           _UpperCAmelCase     :  str							      =    tokenizer_r.save_pretrained(A						,					legacy_format=A       )
                           _UpperCAmelCase     :  str							      =    tokenizer_p.save_pretrained(A       )

                           # Checks it save with the same files
                           self.assertSequenceEqual(A						,					A       )

                           # Checks everything loads correctly in the same way
                           _UpperCAmelCase     :  Optional[int]							      =    tokenizer_r.from_pretrained(A       )
                           _UpperCAmelCase     :  Dict							      =    tokenizer_p.from_pretrained(A       )

                           # Check special tokens are set accordingly on Rust and Python
                           for key in tokenizer_pp.special_tokens_map:
                                  self.assertTrue(hasattr(A						,					A       )       )

                           shutil.rmtree(A       )

                           # Save tokenizer rust, legacy_format=False
                           _UpperCAmelCase     :  Optional[int]							      =    tempfile.mkdtemp()

                           _UpperCAmelCase     :  Optional[int]							      =    tokenizer_r.save_pretrained(A						,					legacy_format=A       )
                           _UpperCAmelCase     :  Dict							      =    tokenizer_p.save_pretrained(A       )

                           # Checks it saved the tokenizer.json file
                           self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files       )       )

                           # Checks everything loads correctly in the same way
                           _UpperCAmelCase     :  List[Any]							      =    tokenizer_r.from_pretrained(A       )
                           _UpperCAmelCase     :  Optional[int]							      =    tokenizer_p.from_pretrained(A       )

                           # Check special tokens are set accordingly on Rust and Python
                           for key in tokenizer_pp.special_tokens_map:
                                  self.assertTrue(hasattr(A						,					A       )       )

                           shutil.rmtree(A       )




      @require_torch
      def      _A	(       self						:				Tuple       ):
             if not self.test_seqaseq:
                    return

             _UpperCAmelCase     :  Union[str, Any]							      =    self.get_tokenizers()
             for tokenizer in tokenizers:
                    with self.subTest(F"""{tokenizer.__class__.__name__}"""       ):
                           # Longer text that will definitely require truncation.
                           _UpperCAmelCase     :  Optional[Any]							      =    [
                               " UN Chief Says There Is No Military Solution in Syria",
                               " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"
                               " Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"
                               " will only worsen the violence and misery for millions of people.",
                           ]
                           _UpperCAmelCase     :  Optional[Any]							      =    [
                               "Şeful ONU declară că nu există o soluţie militară în Siria",
                               "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"
                               " Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"
                               " că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
                           ]
                           try:
                                  _UpperCAmelCase     :  Optional[int]							      =    tokenizer.prepare_seqaseq_batch(
                                      src_texts=A						,					tgt_texts=A						,					max_length=3						,					max_target_length=10						,					return_tensors="pt"						,					src_lang="eng_Latn"						,					tgt_lang="ron_Latn"						,					)
                           except NotImplementedError:
                                  return
                           self.assertEqual(batch.input_ids.shape[1]						,					3       )
                           self.assertEqual(batch.labels.shape[1]						,					10       )
                           # max_target_length will default to max_length if not specified
                           _UpperCAmelCase     :  Tuple							      =    tokenizer.prepare_seqaseq_batch(
                               A						,					tgt_texts=A						,					max_length=3						,					return_tensors="pt"       )
                           self.assertEqual(batch.input_ids.shape[1]						,					3       )
                           self.assertEqual(batch.labels.shape[1]						,					3       )

                           _UpperCAmelCase     :  Union[str, Any]							      =    tokenizer.prepare_seqaseq_batch(
                               src_texts=A						,					max_length=3						,					max_target_length=10						,					return_tensors="pt"       )
                           self.assertEqual(batch_encoder_only.input_ids.shape[1]						,					3       )
                           self.assertEqual(batch_encoder_only.attention_mask.shape[1]						,					3       )
                           self.assertNotIn("decoder_input_ids"						,					A       )




      @unittest.skip("Unfortunately way too slow to build a BPE with SentencePiece."       )
      def      _A	(       self						:				List[Any]       ):
             pass





      def      _A	(       self						:				Union[str, Any]       ):
             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                    with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
                           _UpperCAmelCase     :  Any							      =    [AddedToken("<special>"						,					lstrip=A       )]

                           _UpperCAmelCase     :  List[Any]							      =    self.rust_tokenizer_class.from_pretrained(
                               A						,					additional_special_tokens=A						,					**A       )
                           _UpperCAmelCase     :  Dict							      =    tokenizer_r.encode("Hey this is a <special> token"       )

                           _UpperCAmelCase     :  Any							      =    tokenizer_r.encode("<special>"						,					add_special_tokens=A       )[0]

                           self.assertTrue(special_token_id in r_output       )

                           if self.test_slow_tokenizer:
                                  _UpperCAmelCase     :  Dict							      =    self.rust_tokenizer_class.from_pretrained(
                                      A						,					additional_special_tokens=A						,					**A						,					)
                                  _UpperCAmelCase     :  Optional[int]							      =    self.tokenizer_class.from_pretrained(
                                      A						,					additional_special_tokens=A						,					**A       )

                                  _UpperCAmelCase     :  Union[str, Any]							      =    tokenizer_p.encode("Hey this is a <special> token"       )

                                  _UpperCAmelCase     :  Any							      =    tokenizer_cr.encode("Hey this is a <special> token"       )

                                  self.assertEqual(A						,					A       )
                                  self.assertEqual(A						,					A       )
                                  self.assertTrue(special_token_id in p_output       )
                                  self.assertTrue(special_token_id in cr_output       )

@require_torch
@require_sentencepiece
@require_tokenizers
class 	lowerCamelCase_	(unittest.TestCase    ):
      '''simple docstring'''

      __UpperCamelCase:  Dict						=						"facebook/nllb-200-distilled-600M"
      __UpperCamelCase:  Optional[int]						=						[
          " UN Chief Says There Is No Military Solution in Syria",
          " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
      ]
      __UpperCamelCase:  str						=						[
          "Şeful ONU declară că nu există o soluţie militară în Siria",
          "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
          " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
          " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
      ]
      __UpperCamelCase:  str						=						[
          2_5_6_0_4_7,
          1_6_2_9_7,
          1_3_4_4_0_8,
          8_1_6_5,
          2_4_8_0_6_6,
          1_4_7_3_4,
          9_5_0,
          1_1_3_5,
          1_0_5_7_2_1,
          3_5_7_3,
          8_3,
          2_7_3_5_2,
          1_0_8,
          4_9_4_8_6,
          2,
      ]




      @classmethod
      def      _A	(       cls						:				int       ):
             _UpperCAmelCase     :  NllbTokenizer							      =    NllbTokenizer.from_pretrained(
                 cls.checkpoint_name						,					src_lang="eng_Latn"						,					tgt_lang="ron_Latn"       )
             _UpperCAmelCase     :  Union[str, Any]							      =    1
             return cls




      def      _A	(       self						:				Any       ):
             self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ace_Arab"]						,					256001       )
             self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ace_Latn"]						,					256002       )
             self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["fra_Latn"]						,					256057       )




      def      _A	(       self						:				Union[str, Any]       ):
             _UpperCAmelCase     :  Optional[int]							      =    self.tokenizer.batch_encode_plus(self.src_text       ).input_ids[0]
             self.assertListEqual(self.expected_src_tokens						,					A       )




      def      _A	(       self						:				Tuple       ):
             self.assertIn(A						,					self.tokenizer.all_special_ids       )
             # fmt: off
             _UpperCAmelCase     :  List[Any]							      =    [RO_CODE, 4254, 98068, 112923, 39072, 3909, 713, 102767, 26, 17314, 35642, 14683, 33118, 2022, 66987, 2, 256047]
             # fmt: on

             _UpperCAmelCase     :  Tuple							      =    self.tokenizer.decode(A						,					skip_special_tokens=A       )
             _UpperCAmelCase     :  Optional[Any]							      =    self.tokenizer.decode(generated_ids[1:]						,					skip_special_tokens=A       )
             self.assertEqual(A						,					A       )
             self.assertNotIn(self.tokenizer.eos_token						,					A       )




      def      _A	(       self						:				Optional[int]       ):
             _UpperCAmelCase     :  List[Any]							      =    ["this is gunna be a long sentence " * 20]
             assert isinstance(src_text[0]						,					A       )
             _UpperCAmelCase     :  Dict							      =    10
             _UpperCAmelCase     :  Tuple							      =    self.tokenizer(A						,					max_length=A						,					truncation=A       ).input_ids[0]
             self.assertEqual(ids[-1]						,					2       )
             self.assertEqual(ids[0]						,					A       )
             self.assertEqual(len(A       )						,					A       )




      def      _A	(       self						:				Dict       ):
             self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "ar_AR"]       )						,					[256203, 3]       )




      def      _A	(       self						:				Optional[Any]       ):
             _UpperCAmelCase     :  Dict							      =    tempfile.mkdtemp()
             _UpperCAmelCase     :  str							      =    self.tokenizer.fairseq_tokens_to_ids
             self.tokenizer.save_pretrained(A       )
             _UpperCAmelCase     :  Tuple							      =    NllbTokenizer.from_pretrained(A       )
             self.assertDictEqual(new_tok.fairseq_tokens_to_ids						,					A       )




      @require_torch
      def      _A	(       self						:				Dict       ):
             _UpperCAmelCase     :  List[str]							      =    self.tokenizer(
                 self.src_text						,					text_target=self.tgt_text						,					padding=A						,					truncation=A						,					max_length=len(self.expected_src_tokens       )						,					return_tensors="pt"						,					)
             _UpperCAmelCase     :  Tuple							      =    shift_tokens_right(
                 batch["labels"]						,					self.tokenizer.pad_token_id						,					self.tokenizer.lang_code_to_id["ron_Latn"]       )

             self.assertIsInstance(A						,					A       )

             self.assertEqual((2, 15)						,					batch.input_ids.shape       )
             self.assertEqual((2, 15)						,					batch.attention_mask.shape       )
             _UpperCAmelCase     :  Dict							      =    batch.input_ids.tolist()[0]
             self.assertListEqual(self.expected_src_tokens						,					A       )
             self.assertEqual(A						,					batch.decoder_input_ids[0, 0]       )  # EOS
             # Test that special tokens are reset
             self.assertEqual(self.tokenizer.prefix_tokens						,					[EN_CODE]       )
             self.assertEqual(self.tokenizer.suffix_tokens						,					[self.tokenizer.eos_token_id]       )




      def      _A	(       self						:				str       ):
             _UpperCAmelCase     :  Optional[Any]							      =    self.tokenizer(self.src_text						,					padding=A						,					truncation=A						,					max_length=3						,					return_tensors="pt"       )
             _UpperCAmelCase     :  Dict							      =    self.tokenizer(
                 text_target=self.tgt_text						,					padding=A						,					truncation=A						,					max_length=10						,					return_tensors="pt"       )
             _UpperCAmelCase     :  List[Any]							      =    targets["input_ids"]
             _UpperCAmelCase     :  Union[str, Any]							      =    shift_tokens_right(
                 A						,					self.tokenizer.pad_token_id						,					decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]						,					)

             self.assertEqual(batch.input_ids.shape[1]						,					3       )
             self.assertEqual(batch.decoder_input_ids.shape[1]						,					10       )




      @require_torch
      def      _A	(       self						:				List[Any]       ):
             _UpperCAmelCase     :  str							      =    self.tokenizer._build_translation_inputs(
                 "A test"						,					return_tensors="pt"						,					src_lang="eng_Latn"						,					tgt_lang="fra_Latn"       )

             self.assertEqual(
                 nested_simplify(A       )						,					{
                     # A, test, EOS, en_XX
                     "input_ids": [[256047, 70, 7356, 2]],
                     "attention_mask": [[1, 1, 1, 1]],
                     # ar_AR
                     "forced_bos_token_id": 256057,
                 }						,					)





      @require_torch
      def      _A	(       self						:				Any       ):
             _UpperCAmelCase     :  Dict							      =    True
             _UpperCAmelCase     :  Any							      =    self.tokenizer(
                 "UN Chief says there is no military solution in Syria"						,					src_lang="eng_Latn"						,					tgt_lang="fra_Latn"       )
             self.assertEqual(
                 inputs.input_ids						,					[16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2, 256047]       )

             _UpperCAmelCase     :  Optional[int]							      =    False
             _UpperCAmelCase     :  str							      =    self.tokenizer(
                 "UN Chief says there is no military solution in Syria"						,					src_lang="eng_Latn"						,					tgt_lang="fra_Latn"       )
             self.assertEqual(
                 inputs.input_ids						,					[256047, 16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2]       )

