




import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


UpperCAmelCase_									=	re.compile(r'\b(a|an|the)\b', re.UNICODE)

UpperCAmelCase_									=	None




def 					lowerCamelCase__       (						):





       '''simple docstring'''



       __lowerCamelCase   =						argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0."""	)
       parser.add_argument("""data_file"""  ,			metavar="""data.json"""  ,			help="""Input data JSON file."""	)
       parser.add_argument("""pred_file"""  ,			metavar="""pred.json"""  ,			help="""Model predictions."""	)
       parser.add_argument(
           """--out-file"""  ,			"""-o"""  ,			metavar="""eval.json"""  ,			help="""Write accuracy metrics to file (default is stdout)."""	)
       parser.add_argument(
           """--na-prob-file"""  ,			"""-n"""  ,			metavar="""na_prob.json"""  ,			help="""Model estimates of probability of no answer."""	)
       parser.add_argument(
           """--na-prob-thresh"""  ,			"""-t"""  ,			type=A__  ,			default=1.0  ,			help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""  ,			)
       parser.add_argument(
           """--out-image-dir"""  ,			"""-p"""  ,			metavar="""out_images"""  ,			default=A__  ,			help="""Save precision-recall curves to directory."""	)
       parser.add_argument("""--verbose"""  ,			"""-v"""  ,			action="""store_true"""	)
       if len(sys.argv	) == 1:
              parser.print_help()
              sys.exit(1	)
       return parser.parse_args()




def 					lowerCamelCase__       (						A__       :					Any	):





       '''simple docstring'''



       __lowerCamelCase   =						{}
       for article in dataset:
              for p in article["paragraphs"]:
                     for qa in p["qas"]:
                            __lowerCamelCase   =						bool(qa["""answers"""]["""text"""]	)
       return qid_to_has_ans




def 					lowerCamelCase__       (						A__       :					Union[str, Any]	):





       '''simple docstring'''



       def remove_articles(A__       :					List[Any]	):
              return ARTICLES_REGEX.sub(""" """  ,			A__	)

       def white_space_fix(A__       :					List[Any]	):
              return " ".join(text.split()	)

       def remove_punc(A__       :					Optional[Any]	):
              __lowerCamelCase   =						set(string.punctuation	)
              return "".join(ch for ch in text if ch not in exclude	)

       def lower(A__       :					Dict	):
              return text.lower()

       return white_space_fix(remove_articles(remove_punc(lower(A__	)	)	)	)




def 					lowerCamelCase__       (						A__       :					Union[str, Any]	):





       '''simple docstring'''



       if not s:
              return []
       return normalize_answer(A__	).split()




def 					lowerCamelCase__       (						A__       :					int  ,			A__       :					Any	):





       '''simple docstring'''



       return int(normalize_answer(A__	) == normalize_answer(A__	)	)




def 					lowerCamelCase__       (						A__       :					Dict  ,			A__       :					str	):





       '''simple docstring'''



       __lowerCamelCase   =						get_tokens(A__	)
       __lowerCamelCase   =						get_tokens(A__	)
       __lowerCamelCase   =						collections.Counter(A__	) & collections.Counter(A__	)
       __lowerCamelCase   =						sum(common.values()	)
       if len(A__	) == 0 or len(A__	) == 0:
              # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
              return int(gold_toks == pred_toks	)
       if num_same == 0:
              return 0
       __lowerCamelCase   =						1.0 * num_same / len(A__	)
       __lowerCamelCase   =						1.0 * num_same / len(A__	)
       __lowerCamelCase   =						(2 * precision * recall) / (precision + recall)
       return fa




def 					lowerCamelCase__       (						A__       :					Optional[Any]  ,			A__       :					Dict	):





       '''simple docstring'''



       __lowerCamelCase   =						{}
       __lowerCamelCase   =						{}
       for article in dataset:
              for p in article["paragraphs"]:
                     for qa in p["qas"]:
                            __lowerCamelCase   =						qa["""id"""]
                            __lowerCamelCase   =						[t for t in qa["""answers"""]["""text"""] if normalize_answer(A__	)]
                            if not gold_answers:
                                   # For unanswerable questions, only correct answer is empty string
                                   __lowerCamelCase   =						[""""""]
                            if qid not in preds:
                                   print(f'Missing prediction for {qid}'	)
                                   continue
                            __lowerCamelCase   =						preds[qid]
                            # Take max over all gold answers
                            __lowerCamelCase   =						max(compute_exact(A__  ,			A__	) for a in gold_answers	)
                            __lowerCamelCase   =						max(compute_fa(A__  ,			A__	) for a in gold_answers	)
       return exact_scores, fa_scores




def 					lowerCamelCase__       (						A__       :					Dict  ,			A__       :					str  ,			A__       :					Dict  ,			A__       :					Any	):





       '''simple docstring'''



       __lowerCamelCase   =						{}
       for qid, s in scores.items():
              __lowerCamelCase   =						na_probs[qid] > na_prob_thresh
              if pred_na:
                     __lowerCamelCase   =						float(not qid_to_has_ans[qid]	)
              else:
                     __lowerCamelCase   =						s
       return new_scores




def 					lowerCamelCase__       (						A__       :					Optional[int]  ,			A__       :					Tuple  ,			A__       :					str=None	):





       '''simple docstring'''



       if not qid_list:
              __lowerCamelCase   =						len(A__	)
              return collections.OrderedDict(
                  [
                      ("""exact""", 100.0 * sum(exact_scores.values()	) / total),
                      ("""f1""", 100.0 * sum(fa_scores.values()	) / total),
                      ("""total""", total),
                  ]	)
       else:
              __lowerCamelCase   =						len(A__	)
              return collections.OrderedDict(
                  [
                      ("""exact""", 100.0 * sum(exact_scores[k] for k in qid_list	) / total),
                      ("""f1""", 100.0 * sum(fa_scores[k] for k in qid_list	) / total),
                      ("""total""", total),
                  ]	)




def 					lowerCamelCase__       (						A__       :					List[Any]  ,			A__       :					Union[str, Any]  ,			A__       :					Tuple	):





       '''simple docstring'''



       for k in new_eval:
              __lowerCamelCase   =						new_eval[k]




def 					lowerCamelCase__       (						A__       :					Tuple  ,			A__       :					int  ,			A__       :					Dict  ,			A__       :					Optional[int]	):





       '''simple docstring'''



       plt.step(A__  ,			A__  ,			color="""b"""  ,			alpha=0.2  ,			where="""post"""	)
       plt.fill_between(A__  ,			A__  ,			step="""post"""  ,			alpha=0.2  ,			color="""b"""	)
       plt.xlabel("""Recall"""	)
       plt.ylabel("""Precision"""	)
       plt.xlim([0.0, 1.05]	)
       plt.ylim([0.0, 1.05]	)
       plt.title(A__	)
       plt.savefig(A__	)
       plt.clf()




def 					lowerCamelCase__       (						A__       :					Dict  ,			A__       :					List[str]  ,			A__       :					int  ,			A__       :					Dict  ,			A__       :					Any=None  ,			A__       :					Optional[Any]=None	):





       '''simple docstring'''



       __lowerCamelCase   =						sorted(A__  ,			key=lambda A__	: na_probs[k]	)
       __lowerCamelCase   =						0.0
       __lowerCamelCase   =						1.0
       __lowerCamelCase   =						0.0
       __lowerCamelCase   =						[1.0]
       __lowerCamelCase   =						[0.0]
       __lowerCamelCase   =						0.0
       for i, qid in enumerate(A__	):
              if qid_to_has_ans[qid]:
                     true_pos += scores[qid]
              __lowerCamelCase   =						true_pos / float(i + 1	)
              __lowerCamelCase   =						true_pos / float(A__	)
              if i == len(A__	) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
                     # i.e., if we can put a threshold after this point
                     avg_prec += cur_p * (cur_r - recalls[-1])
                     precisions.append(A__	)
                     recalls.append(A__	)
       if out_image:
              plot_pr_curve(A__  ,			A__  ,			A__  ,			A__	)
       return {"ap": 100.0 * avg_prec}




def 					lowerCamelCase__       (						A__       :					str  ,			A__       :					Union[str, Any]  ,			A__       :					Optional[int]  ,			A__       :					int  ,			A__       :					Tuple  ,			A__       :					Optional[Any]	):





       '''simple docstring'''



       if out_image_dir and not os.path.exists(A__	):
              os.makedirs(A__	)
       __lowerCamelCase   =						sum(1 for v in qid_to_has_ans.values() if v	)
       if num_true_pos == 0:
              return
       __lowerCamelCase   =						make_precision_recall_eval(
           A__  ,			A__  ,			A__  ,			A__  ,			out_image=os.path.join(A__  ,			"""pr_exact.png"""	)  ,			title="""Precision-Recall curve for Exact Match score"""  ,			)
       __lowerCamelCase   =						make_precision_recall_eval(
           A__  ,			A__  ,			A__  ,			A__  ,			out_image=os.path.join(A__  ,			"""pr_f1.png"""	)  ,			title="""Precision-Recall curve for F1 score"""  ,			)
       __lowerCamelCase   =						{k: float(A__	) for k, v in qid_to_has_ans.items()}
       __lowerCamelCase   =						make_precision_recall_eval(
           A__  ,			A__  ,			A__  ,			A__  ,			out_image=os.path.join(A__  ,			"""pr_oracle.png"""	)  ,			title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""  ,			)
       merge_eval(A__  ,			A__  ,			"""pr_exact"""	)
       merge_eval(A__  ,			A__  ,			"""pr_f1"""	)
       merge_eval(A__  ,			A__  ,			"""pr_oracle"""	)




def 					lowerCamelCase__       (						A__       :					Optional[Any]  ,			A__       :					Union[str, Any]  ,			A__       :					int  ,			A__       :					int	):





       '''simple docstring'''



       if not qid_list:
              return
       __lowerCamelCase   =						[na_probs[k] for k in qid_list]
       __lowerCamelCase   =						np.ones_like(A__	) / float(len(A__	)	)
       plt.hist(A__  ,			weights=A__  ,			bins=20  ,			range=(0.0, 1.0)	)
       plt.xlabel("""Model probability of no-answer"""	)
       plt.ylabel("""Proportion of dataset"""	)
       plt.title(f'Histogram of no-answer probability: {name}'	)
       plt.savefig(os.path.join(A__  ,			f'na_prob_hist_{name}.png'	)	)
       plt.clf()




def 					lowerCamelCase__       (						A__       :					Union[str, Any]  ,			A__       :					Optional[int]  ,			A__       :					Tuple  ,			A__       :					int	):





       '''simple docstring'''



       __lowerCamelCase   =						sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]	)
       __lowerCamelCase   =						num_no_ans
       __lowerCamelCase   =						cur_score
       __lowerCamelCase   =						0.0
       __lowerCamelCase   =						sorted(A__  ,			key=lambda A__	: na_probs[k]	)
       for i, qid in enumerate(A__	):
              if qid not in scores:
                     continue
              if qid_to_has_ans[qid]:
                     __lowerCamelCase   =						scores[qid]
              else:
                     if preds[qid]:
                            __lowerCamelCase   =						-1
                     else:
                            __lowerCamelCase   =						0
              cur_score += diff
              if cur_score > best_score:
                     __lowerCamelCase   =						cur_score
                     __lowerCamelCase   =						na_probs[qid]
       return 100.0 * best_score / len(A__	), best_thresh




def 					lowerCamelCase__       (						A__       :					str  ,			A__       :					List[str]  ,			A__       :					Optional[Any]  ,			A__       :					Any  ,			A__       :					str  ,			A__       :					int	):





       '''simple docstring'''



       __lowerCamelCase,   __lowerCamelCase   =						find_best_thresh(A__  ,			A__  ,			A__  ,			A__	)
       __lowerCamelCase,   __lowerCamelCase   =						find_best_thresh(A__  ,			A__  ,			A__  ,			A__	)
       __lowerCamelCase   =						best_exact
       __lowerCamelCase   =						exact_thresh
       __lowerCamelCase   =						best_fa
       __lowerCamelCase   =						fa_thresh




def 					lowerCamelCase__       (						):





       '''simple docstring'''



       with open(OPTS.data_file	) as f:
              __lowerCamelCase   =						json.load(A__	)
              __lowerCamelCase   =						dataset_json["""data"""]
       with open(OPTS.pred_file	) as f:
              __lowerCamelCase   =						json.load(A__	)
       if OPTS.na_prob_file:
              with open(OPTS.na_prob_file	) as f:
                     __lowerCamelCase   =						json.load(A__	)
       else:
              __lowerCamelCase   =						{k: 0.0 for k in preds}
       __lowerCamelCase   =						make_qid_to_has_ans(A__	)  # maps qid to True/False
       __lowerCamelCase   =						[k for k, v in qid_to_has_ans.items() if v]
       __lowerCamelCase   =						[k for k, v in qid_to_has_ans.items() if not v]
       __lowerCamelCase,   __lowerCamelCase   =						get_raw_scores(A__  ,			A__	)
       __lowerCamelCase   =						apply_no_ans_threshold(A__  ,			A__  ,			A__  ,			OPTS.na_prob_thresh	)
       __lowerCamelCase   =						apply_no_ans_threshold(A__  ,			A__  ,			A__  ,			OPTS.na_prob_thresh	)
       __lowerCamelCase   =						make_eval_dict(A__  ,			A__	)
       if has_ans_qids:
              __lowerCamelCase   =						make_eval_dict(A__  ,			A__  ,			qid_list=A__	)
              merge_eval(A__  ,			A__  ,			"""HasAns"""	)
       if no_ans_qids:
              __lowerCamelCase   =						make_eval_dict(A__  ,			A__  ,			qid_list=A__	)
              merge_eval(A__  ,			A__  ,			"""NoAns"""	)
       if OPTS.na_prob_file:
              find_all_best_thresh(A__  ,			A__  ,			A__  ,			A__  ,			A__  ,			A__	)
       if OPTS.na_prob_file and OPTS.out_image_dir:
              run_precision_recall_analysis(A__  ,			A__  ,			A__  ,			A__  ,			A__  ,			OPTS.out_image_dir	)
              histogram_na_prob(A__  ,			A__  ,			OPTS.out_image_dir  ,			"""hasAns"""	)
              histogram_na_prob(A__  ,			A__  ,			OPTS.out_image_dir  ,			"""noAns"""	)
       if OPTS.out_file:
              with open(OPTS.out_file  ,			"""w"""	) as f:
                     json.dump(A__  ,			A__	)
       else:
              print(json.dumps(A__  ,			indent=2	)	)


if __name__ == "__main__":
       UpperCAmelCase_									=	parse_args()
       if OPTS.out_image_dir:
              import matplotlib

              matplotlib.use('Agg')
              import matplotlib.pyplot as plt
       main()







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available


UpperCAmelCase_									=	{}


try:
       if not is_sentencepiece_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       UpperCAmelCase_									=	['MLukeTokenizer']

if TYPE_CHECKING:
       try:
              if not is_sentencepiece_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .tokenization_mluke import MLukeTokenizer


else:
       import sys

       UpperCAmelCase_									=	_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)

