






"""simple docstring"""


import re
import tempfile
import unittest
from pathlib import Path

from datasets.utils.metadata import DatasetMetadata



def    lowerCAmelCase							(       __UpperCamelCase					):




  """simple docstring"""







  __A		     =					min(re.search('''^ +'''						,			__UpperCamelCase					).end() if t.startswith(''' '''					) else 0 for t in string.splitlines()					)
  return "\n".join([line[indent_level:] for line in string.splitlines() if indent_level < len(__UpperCamelCase					)]					)


lowercase_								=	'\\n---\nlanguage:\n- zh\n- en\ntask_ids:\n- sentiment-classification\n---\n# Begin of markdown\n\nSome cool dataset card\n'

lowercase_								=	'\\n---\n---\n# Begin of markdown\n\nSome cool dataset card\n'


lowercase_								=	'\\n# Begin of markdown\n\nSome cool dataset card\n'





class    snake_case						(				unittest.TestCase		):


   '''simple docstring'''

   def 							_SCREAMING_SNAKE_CASE							(      self					:	List[str]    ):

     '''simple docstring'''
     with tempfile.TemporaryDirectory() as tmp_dir:
       __A		     =					Path(_lowerCamelCase    ) / '''README.md'''
       with open(_lowerCamelCase,			'''w+'''    ) as readme_file:
         readme_file.write(_lowerCamelCase    )
       __A		     =					DatasetMetadata.from_readme(_lowerCamelCase    )
       self.assertDictEqual(_lowerCamelCase,			{'''language''': ['''zh''', '''en'''], '''task_ids''': ['''sentiment-classification''']}    )

       with open(_lowerCamelCase,			'''w+'''    ) as readme_file:
         readme_file.write(_lowerCamelCase    )
       __A		     =					DatasetMetadata.from_readme(_lowerCamelCase    )
       self.assertDictEqual(_lowerCamelCase,			{}    )

       with open(_lowerCamelCase,			'''w+'''    ) as readme_file:
         readme_file.write(_lowerCamelCase    )
       __A		     =					DatasetMetadata.from_readme(_lowerCamelCase    )
       self.assertEqual(_lowerCamelCase,			{}    )






   def 							_SCREAMING_SNAKE_CASE							(      self					:	Union[str, Any]    ):

     '''simple docstring'''
     __A		     =					_dedent(
         '''\
            annotations_creators:
            - found
            language_creators:
            - found
            language:
            - en
            license:
            - unknown
            multilinguality:
            - monolingual
            pretty_name: Test Dataset
            size_categories:
            - 10K<n<100K
            source_datasets:
            - extended|other-yahoo-webscope-l6
            task_categories:
            - question-answering
            task_ids:
            - open-domain-qa
            '''    )
     assert DatasetMetadata.from_yaml_string(_lowerCamelCase    )

     __A		     =					_dedent(
         '''\
            annotations_creators:
            - found
            language:
            - en
            license:
            - unknown
            multilinguality:
            - monolingual
            pretty_name: Test Dataset
            size_categories:
            - 10K<n<100K
            source_datasets:
            - extended|other-yahoo-webscope-l6
            task_categories:
            - question-answering
            task_ids:
            - open-domain-qa
            task_ids:
            - open-domain-qa
            '''    )
     with self.assertRaises(_lowerCamelCase    ):
       DatasetMetadata.from_yaml_string(_lowerCamelCase    )

     __A		     =					_dedent(
         '''\
            annotations_creators:
            - found
            language_creators:
            - found
            language:
            - en
            license:
            - unknown
            multilinguality:
            - monolingual
            pretty_name: Test Dataset
            size_categories:
            - 10K<n<100K
            source_datasets:
            - extended|other-yahoo-webscope-l6
            task_categories:
            - text-classification
            task_ids:
            - multi-class-classification
            paperswithcode_id:
            - squad
            configs:
            - en
            train-eval-index:
            - config: en
              task: text-classification
              task_id: multi_class_classification
              splits:
                train_split: train
                eval_split: test
              col_mapping:
                text: text
                label: target
              metrics:
                - type: accuracy
                  name: Accuracy
            extra_gated_prompt: |
              By clicking on “Access repository” below, you also agree to ImageNet Terms of Access:
              [RESEARCHER_FULLNAME] (the "Researcher") has requested permission to use the ImageNet database (the "Database") at Princeton University and Stanford University. In exchange for such permission, Researcher hereby agrees to the following terms and conditions:
              1. Researcher shall use the Database only for non-commercial research and educational purposes.
            extra_gated_fields:
              Company: text
              Country: text
              I agree to use this model for non-commerical use ONLY: checkbox
            '''    )
     assert DatasetMetadata.from_yaml_string(_lowerCamelCase    )










import argparse
import json
import re
import string
import sys

import numpy as np


UpperCAmelCase_   							=      0.5




def    lowerCamelCase__			(		A__			:   Optional[Any]    ,     A__			:   Union[str, Any]      ):





       '''simple docstring'''



       __lowerCamelCase  =		[""".""", """,""", """;""", """:"""]

       for token in remove_tokens:
              __lowerCamelCase  =		ground_truth.replace(A__    ,     """"""      )
              __lowerCamelCase  =		prediction.replace(A__    ,     """"""      )

       __lowerCamelCase,  __lowerCamelCase  =		ground_truth.lower(), prediction.lower()
       __lowerCamelCase,  __lowerCamelCase  =		ground_truth.replace("""/"""    ,     """ """      ), prediction.replace("""/"""    ,     """ """      )
       __lowerCamelCase,  __lowerCamelCase  =		set(ground_truth.split(""" """      )      ), set(prediction.split(""" """      )      )

       __lowerCamelCase  =		ground_truth.intersection(A__      )
       __lowerCamelCase  =		ground_truth.union(A__      )
       __lowerCamelCase  =		len(A__      ) / len(A__      )
       return jaccard




def    lowerCamelCase__			(		A__			:   Optional[Any]      ):





       '''simple docstring'''



       def remove_articles(A__			:   Tuple      ):
              return re.sub(R"""\b(a|an|the)\b"""    ,     """ """    ,     A__      )

       def white_space_fix(A__			:   Optional[Any]      ):
              return " ".join(text.split()      )

       def remove_punc(A__			:   Optional[Any]      ):
              __lowerCamelCase  =		set(string.punctuation      )
              return "".join(ch for ch in text if ch not in exclude      )

       def lower(A__			:   Tuple      ):
              return text.lower()

       return white_space_fix(remove_articles(remove_punc(lower(A__      )      )      )      )




def    lowerCamelCase__			(		A__			:   Any    ,     A__			:   Optional[Any]    ,     A__			:   Optional[Any]      ):





       '''simple docstring'''



       __lowerCamelCase,  __lowerCamelCase,  __lowerCamelCase  =		0, 0, 0

       __lowerCamelCase  =		"""Parties""" in qa_id

       # first check if ground truth is empty
       if len(A__      ) == 0:
              if len(A__      ) > 0:
                     fp += len(A__      )  # false positive for each one
       else:
              for ground_truth in ground_truths:
                     assert len(A__      ) > 0
                     # check if there is a match
                     __lowerCamelCase  =		False
                     for pred in predictions:
                            if substr_ok:
                                   __lowerCamelCase  =		get_jaccard(A__    ,     A__      ) >= IOU_THRESH or ground_truth in pred
                            else:
                                   __lowerCamelCase  =		get_jaccard(A__    ,     A__      ) >= IOU_THRESH
                            if is_match:
                                   __lowerCamelCase  =		True

                     if match_found:
                            tp += 1
                     else:
                            fn += 1

        # now also get any fps by looping through preds
              for pred in predictions:
                     # Check if there's a match. if so, don't count (don't want to double count based on the above)
                     # but if there's no match, then this is a false positive.
                     # (Note: we get the true positives in the above loop instead of this loop so that we don't double count
                     # multiple predictions that are matched with the same answer.)
                     __lowerCamelCase  =		False
                     for ground_truth in ground_truths:
                            assert len(A__      ) > 0
                            if substr_ok:
                                   __lowerCamelCase  =		get_jaccard(A__    ,     A__      ) >= IOU_THRESH or ground_truth in pred
                            else:
                                   __lowerCamelCase  =		get_jaccard(A__    ,     A__      ) >= IOU_THRESH
                            if is_match:
                                   __lowerCamelCase  =		True

                     if not match_found:
                            fp += 1

       __lowerCamelCase  =		tp / (tp + fp) if tp + fp > 0 else np.nan
       __lowerCamelCase  =		tp / (tp + fn) if tp + fn > 0 else np.nan

       return precision, recall




def    lowerCamelCase__			(		A__			:   int      ):





       '''simple docstring'''



       __lowerCamelCase  =		precisions[::-1]
       for i in range(1    ,     len(A__      )      ):
              __lowerCamelCase  =		max(precision_best[i - 1]    ,     precision_best[i]      )
       __lowerCamelCase  =		precision_best[::-1]
       return precisions




def    lowerCamelCase__			(		A__			:   str    ,     A__			:   Union[str, Any]      ):





       '''simple docstring'''



       __lowerCamelCase  =		process_precisions(A__      )
       __lowerCamelCase  =		np.trapz(A__    ,     A__      )
       if np.isnan(A__      ):
              return 0
       return aupr




def    lowerCamelCase__			(		A__			:   Optional[Any]    ,     A__			:   List[str]    ,     A__			:   int      ):





       '''simple docstring'''



       __lowerCamelCase  =		process_precisions(A__      )
       __lowerCamelCase  =		0
       for prec, recall in zip(A__    ,     A__      ):
              if recall >= recall_thresh:
                     __lowerCamelCase  =		prec
                     break
       return prec_at_recall




def    lowerCamelCase__			(		A__			:   Optional[Any]    ,     A__			:   List[Any]      ):





       '''simple docstring'''



       return normalize_answer(A__      ) == normalize_answer(A__      )




def    lowerCamelCase__			(		A__			:   Tuple    ,     A__			:   int    ,     A__			:   Dict      ):





       '''simple docstring'''



       __lowerCamelCase  =		0
       for pred in predictions:
              for ground_truth in ground_truths:
                     __lowerCamelCase  =		metric_fn(A__    ,     A__      )
                     if score == 1:  # break the loop when one prediction matches the ground truth
                            break
              if score == 1:
                     break
       return score




def    lowerCamelCase__			(		A__			:   Union[str, Any]    ,     A__			:   Optional[Any]      ):





       '''simple docstring'''



       __lowerCamelCase  =		__lowerCamelCase  =		__lowerCamelCase  =		0
       __lowerCamelCase  =		[]
       __lowerCamelCase  =		[]
       for article in dataset:
              for paragraph in article["paragraphs"]:
                     for qa in paragraph["qas"]:
                            total += 1
                            if qa["id"] not in predictions:
                                   __lowerCamelCase  =		"""Unanswered question """ + qa["""id"""] + """ will receive score 0."""
                                   print(A__    ,     file=sys.stderr      )
                                   continue
                            __lowerCamelCase  =		[x["""text"""] for x in qa["""answers"""]]
                            __lowerCamelCase  =		predictions[qa["""id"""]]
                            __lowerCamelCase,  __lowerCamelCase  =		compute_precision_recall(A__    ,     A__    ,     qa["""id"""]      )

                            precisions.append(A__      )
                            recalls.append(A__      )

                            if precision == 0 and recall == 0:
                                   fa += 0
                            else:
                                   fa += 2 * (precision * recall) / (precision + recall)

                            exact_match += metric_max_over_ground_truths(A__    ,     A__    ,     A__      )

       __lowerCamelCase  =		[x for _, x in sorted(zip(A__    ,     A__      )      )]
       recalls.sort()

       __lowerCamelCase  =		100.0 * fa / total
       __lowerCamelCase  =		100.0 * exact_match / total
       __lowerCamelCase  =		get_aupr(A__    ,     A__      )

       __lowerCamelCase  =		get_prec_at_recall(A__    ,     A__    ,     recall_thresh=0.9      )
       __lowerCamelCase  =		get_prec_at_recall(A__    ,     A__    ,     recall_thresh=0.8      )

       return {
           "exact_match": exact_match,
           "f1": fa,
           "aupr": aupr,
           "prec_at_80_recall": prec_at_aa_recall,
           "prec_at_90_recall": prec_at_aa_recall,
       }


if __name__ == "__main__":
       UpperCAmelCase_   							=      argparse.ArgumentParser(description='Evaluation for CUAD')
       parser.add_argument('dataset_file', help='Dataset file')
       parser.add_argument('prediction_file', help='Prediction File')
       UpperCAmelCase_   							=      parser.parse_args()
       with open(args.dataset_file) as dataset_file:
              UpperCAmelCase_   							=      json.load(dataset_file)
              UpperCAmelCase_   							=      dataset_json['data']
       with open(args.prediction_file) as prediction_file:
              UpperCAmelCase_   							=      json.load(prediction_file)
       print(json.dumps(evaluate(dataset, predictions)))

