




import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


SCREAMING_SNAKE_CASE__       :      str  =   re.compile(r'\b(a|an|the)\b', re.UNICODE)

SCREAMING_SNAKE_CASE__       :      List[str]  =   None

def      A      (      )	-> List[Any]:
       lowerCamelCase   :      Optional[int]			 =					argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0."    )
       parser.add_argument("data_file" ,metavar="data.json" ,help="Input data JSON file."    )
       parser.add_argument("pred_file" ,metavar="pred.json" ,help="Model predictions."    )
       parser.add_argument(
           "--out-file" ,"-o" ,metavar="eval.json" ,help="Write accuracy metrics to file (default is stdout)."    )
       parser.add_argument(
           "--na-prob-file" ,"-n" ,metavar="na_prob.json" ,help="Model estimates of probability of no answer."    )
       parser.add_argument(
           "--na-prob-thresh" ,"-t" ,type=_SCREAMING_SNAKE_CASE ,default=1.0 ,help="Predict \"\" if no-answer probability exceeds this (default = 1.0)." ,)
       parser.add_argument(
           "--out-image-dir" ,"-p" ,metavar="out_images" ,default=_SCREAMING_SNAKE_CASE ,help="Save precision-recall curves to directory."    )
       parser.add_argument("--verbose" ,"-v" ,action="store_true"    )
       if len(sys.argv    ) == 1:
              parser.print_help()
              sys.exit(1    )
       return parser.parse_args()

def      A      (      _SCREAMING_SNAKE_CASE    )	-> Optional[Any]:
       lowerCamelCase   :      Optional[int]			 =					{}
       for article in dataset:
              for p in article["paragraphs"]:
                     for qa in p["qas"]:
                            lowerCamelCase   :      Tuple			 =					bool(qa["answers"]["text"]    )
       return qid_to_has_ans

def      A      (      _SCREAMING_SNAKE_CASE    )	-> Union[str, Any]:

       def remove_articles(_SCREAMING_SNAKE_CASE    ):
              return ARTICLES_REGEX.sub(" " ,_SCREAMING_SNAKE_CASE    )

       def white_space_fix(_SCREAMING_SNAKE_CASE    ):
              return " ".join(text.split()    )

       def remove_punc(_SCREAMING_SNAKE_CASE    ):
              lowerCamelCase   :      Tuple			 =					set(string.punctuation    )
              return "".join(ch for ch in text if ch not in exclude    )

       def lower(_SCREAMING_SNAKE_CASE    ):
              return text.lower()

       return white_space_fix(remove_articles(remove_punc(lower(_SCREAMING_SNAKE_CASE    )    )    )    )

def      A      (      _SCREAMING_SNAKE_CASE    )	-> Optional[int]:
       if not s:
              return []
       return normalize_answer(_SCREAMING_SNAKE_CASE    ).split()

def      A      (      _SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE    )	-> Optional[int]:
       return int(normalize_answer(_SCREAMING_SNAKE_CASE    ) == normalize_answer(_SCREAMING_SNAKE_CASE    )    )

def      A      (      _SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE    )	-> Tuple:
       lowerCamelCase   :      str			 =					get_tokens(_SCREAMING_SNAKE_CASE    )
       lowerCamelCase   :      List[Any]			 =					get_tokens(_SCREAMING_SNAKE_CASE    )
       lowerCamelCase   :      List[str]			 =					collections.Counter(_SCREAMING_SNAKE_CASE    ) & collections.Counter(_SCREAMING_SNAKE_CASE    )
       lowerCamelCase   :      Any			 =					sum(common.values()    )
       if len(_SCREAMING_SNAKE_CASE    ) == 0 or len(_SCREAMING_SNAKE_CASE    ) == 0:
              # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
              return int(gold_toks == pred_toks    )
       if num_same == 0:
              return 0
       lowerCamelCase   :      List[str]			 =					1.0 * num_same / len(_SCREAMING_SNAKE_CASE    )
       lowerCamelCase   :      str			 =					1.0 * num_same / len(_SCREAMING_SNAKE_CASE    )
       lowerCamelCase   :      List[Any]			 =					(2 * precision * recall) / (precision + recall)
       return fa

def      A      (      _SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE    )	-> List[str]:
       lowerCamelCase   :      Any			 =					{}
       lowerCamelCase   :      List[Any]			 =					{}
       for article in dataset:
              for p in article["paragraphs"]:
                     for qa in p["qas"]:
                            lowerCamelCase   :      Dict			 =					qa["id"]
                            lowerCamelCase   :      List[str]			 =					[t for t in qa["answers"]["text"] if normalize_answer(_SCREAMING_SNAKE_CASE    )]
                            if not gold_answers:
                                   # For unanswerable questions, only correct answer is empty string
                                   lowerCamelCase   :      List[Any]			 =					[""]
                            if qid not in preds:
                                   print(f'''Missing prediction for {qid}'''    )
                                   continue
                            lowerCamelCase   :      Any			 =					preds[qid]
                            # Take max over all gold answers
                            lowerCamelCase   :      Dict			 =					max(compute_exact(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE    ) for a in gold_answers    )
                            lowerCamelCase   :      Optional[int]			 =					max(compute_fa(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE    ) for a in gold_answers    )
       return exact_scores, fa_scores

def      A      (      _SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE    )	-> Union[str, Any]:
       lowerCamelCase   :      int			 =					{}
       for qid, s in scores.items():
              lowerCamelCase   :      List[Any]			 =					na_probs[qid] > na_prob_thresh
              if pred_na:
                     lowerCamelCase   :      Union[str, Any]			 =					float(not qid_to_has_ans[qid]    )
              else:
                     lowerCamelCase   :      Tuple			 =					s
       return new_scores

def      A      (      _SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE=None    )	-> str:
       if not qid_list:
              lowerCamelCase   :      List[str]			 =					len(_SCREAMING_SNAKE_CASE    )
              return collections.OrderedDict(
                  [
                      ("exact", 100.0 * sum(exact_scores.values()    ) / total),
                      ("f1", 100.0 * sum(fa_scores.values()    ) / total),
                      ("total", total),
                  ]    )
       else:
              lowerCamelCase   :      Union[str, Any]			 =					len(_SCREAMING_SNAKE_CASE    )
              return collections.OrderedDict(
                  [
                      ("exact", 100.0 * sum(exact_scores[k] for k in qid_list    ) / total),
                      ("f1", 100.0 * sum(fa_scores[k] for k in qid_list    ) / total),
                      ("total", total),
                  ]    )

def      A      (      _SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE    )	-> Any:
       for k in new_eval:
              lowerCamelCase   :      Tuple			 =					new_eval[k]

def      A      (      _SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE    )	-> List[Any]:
       plt.step(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,color="b" ,alpha=0.2 ,where="post"    )
       plt.fill_between(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,step="post" ,alpha=0.2 ,color="b"    )
       plt.xlabel("Recall"    )
       plt.ylabel("Precision"    )
       plt.xlim([0.0, 1.05]    )
       plt.ylim([0.0, 1.05]    )
       plt.title(_SCREAMING_SNAKE_CASE    )
       plt.savefig(_SCREAMING_SNAKE_CASE    )
       plt.clf()

def      A      (      _SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE=None ,_SCREAMING_SNAKE_CASE=None    )	-> List[str]:
       lowerCamelCase   :      Tuple			 =					sorted(_SCREAMING_SNAKE_CASE ,key=lambda _SCREAMING_SNAKE_CASE    : na_probs[k]    )
       lowerCamelCase   :      List[Any]			 =					0.0
       lowerCamelCase   :      Union[str, Any]			 =					1.0
       lowerCamelCase   :      int			 =					0.0
       lowerCamelCase   :      Union[str, Any]			 =					[1.0]
       lowerCamelCase   :      str			 =					[0.0]
       lowerCamelCase   :      int			 =					0.0
       for i, qid in enumerate(_SCREAMING_SNAKE_CASE    ):
              if qid_to_has_ans[qid]:
                     true_pos += scores[qid]
              lowerCamelCase   :      Optional[int]			 =					true_pos / float(i + 1    )
              lowerCamelCase   :      Optional[int]			 =					true_pos / float(_SCREAMING_SNAKE_CASE    )
              if i == len(_SCREAMING_SNAKE_CASE    ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
                     # i.e., if we can put a threshold after this point
                     avg_prec += cur_p * (cur_r - recalls[-1])
                     precisions.append(_SCREAMING_SNAKE_CASE    )
                     recalls.append(_SCREAMING_SNAKE_CASE    )
       if out_image:
              plot_pr_curve(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE    )
       return {"ap": 100.0 * avg_prec}

def      A      (      _SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE    )	-> Union[str, Any]:
       if out_image_dir and not os.path.exists(_SCREAMING_SNAKE_CASE    ):
              os.makedirs(_SCREAMING_SNAKE_CASE    )
       lowerCamelCase   :      Union[str, Any]			 =					sum(1 for v in qid_to_has_ans.values() if v    )
       if num_true_pos == 0:
              return
       lowerCamelCase   :      int			 =					make_precision_recall_eval(
           _SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,out_image=os.path.join(_SCREAMING_SNAKE_CASE ,"pr_exact.png"    ) ,title="Precision-Recall curve for Exact Match score" ,)
       lowerCamelCase   :      Optional[int]			 =					make_precision_recall_eval(
           _SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,out_image=os.path.join(_SCREAMING_SNAKE_CASE ,"pr_f1.png"    ) ,title="Precision-Recall curve for F1 score" ,)
       lowerCamelCase   :      Union[str, Any]			 =					{k: float(_SCREAMING_SNAKE_CASE    ) for k, v in qid_to_has_ans.items()}
       lowerCamelCase   :      List[str]			 =					make_precision_recall_eval(
           _SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,out_image=os.path.join(_SCREAMING_SNAKE_CASE ,"pr_oracle.png"    ) ,title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)" ,)
       merge_eval(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,"pr_exact"    )
       merge_eval(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,"pr_f1"    )
       merge_eval(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,"pr_oracle"    )

def      A      (      _SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE    )	-> int:
       if not qid_list:
              return
       lowerCamelCase   :      Any			 =					[na_probs[k] for k in qid_list]
       lowerCamelCase   :      Optional[Any]			 =					np.ones_like(_SCREAMING_SNAKE_CASE    ) / float(len(_SCREAMING_SNAKE_CASE    )    )
       plt.hist(_SCREAMING_SNAKE_CASE ,weights=_SCREAMING_SNAKE_CASE ,bins=20 ,range=(0.0, 1.0)    )
       plt.xlabel("Model probability of no-answer"    )
       plt.ylabel("Proportion of dataset"    )
       plt.title(f'''Histogram of no-answer probability: {name}'''    )
       plt.savefig(os.path.join(_SCREAMING_SNAKE_CASE ,f'''na_prob_hist_{name}.png'''    )    )
       plt.clf()

def      A      (      _SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE    )	-> Dict:
       lowerCamelCase   :      List[str]			 =					sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]    )
       lowerCamelCase   :      Dict			 =					num_no_ans
       lowerCamelCase   :      List[Any]			 =					cur_score
       lowerCamelCase   :      Optional[int]			 =					0.0
       lowerCamelCase   :      Union[str, Any]			 =					sorted(_SCREAMING_SNAKE_CASE ,key=lambda _SCREAMING_SNAKE_CASE    : na_probs[k]    )
       for i, qid in enumerate(_SCREAMING_SNAKE_CASE    ):
              if qid not in scores:
                     continue
              if qid_to_has_ans[qid]:
                     lowerCamelCase   :      Any			 =					scores[qid]
              else:
                     if preds[qid]:
                            lowerCamelCase   :      Union[str, Any]			 =					-1
                     else:
                            lowerCamelCase   :      Optional[Any]			 =					0
              cur_score += diff
              if cur_score > best_score:
                     lowerCamelCase   :      Dict			 =					cur_score
                     lowerCamelCase   :      Optional[int]			 =					na_probs[qid]
       return 100.0 * best_score / len(_SCREAMING_SNAKE_CASE    ), best_thresh

def      A      (      _SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE    )	-> Tuple:
       lowerCamelCase ,      lowerCamelCase   :      List[Any]			 =					find_best_thresh(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE    )
       lowerCamelCase ,      lowerCamelCase   :      Dict			 =					find_best_thresh(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE    )
       lowerCamelCase   :      Optional[int]			 =					best_exact
       lowerCamelCase   :      str			 =					exact_thresh
       lowerCamelCase   :      Union[str, Any]			 =					best_fa
       lowerCamelCase   :      Union[str, Any]			 =					fa_thresh

def      A      (      )	-> Union[str, Any]:
       with open(OPTS.data_file    ) as f:
              lowerCamelCase   :      Tuple			 =					json.load(_SCREAMING_SNAKE_CASE    )
              lowerCamelCase   :      List[Any]			 =					dataset_json["data"]
       with open(OPTS.pred_file    ) as f:
              lowerCamelCase   :      Optional[Any]			 =					json.load(_SCREAMING_SNAKE_CASE    )
       if OPTS.na_prob_file:
              with open(OPTS.na_prob_file    ) as f:
                     lowerCamelCase   :      Optional[int]			 =					json.load(_SCREAMING_SNAKE_CASE    )
       else:
              lowerCamelCase   :      List[Any]			 =					{k: 0.0 for k in preds}
       lowerCamelCase   :      Optional[Any]			 =					make_qid_to_has_ans(_SCREAMING_SNAKE_CASE    )  # maps qid to True/False
       lowerCamelCase   :      Tuple			 =					[k for k, v in qid_to_has_ans.items() if v]
       lowerCamelCase   :      Any			 =					[k for k, v in qid_to_has_ans.items() if not v]
       lowerCamelCase ,      lowerCamelCase   :      Optional[Any]			 =					get_raw_scores(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE    )
       lowerCamelCase   :      str			 =					apply_no_ans_threshold(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,OPTS.na_prob_thresh    )
       lowerCamelCase   :      Optional[int]			 =					apply_no_ans_threshold(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,OPTS.na_prob_thresh    )
       lowerCamelCase   :      Tuple			 =					make_eval_dict(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE    )
       if has_ans_qids:
              lowerCamelCase   :      List[Any]			 =					make_eval_dict(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,qid_list=_SCREAMING_SNAKE_CASE    )
              merge_eval(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,"HasAns"    )
       if no_ans_qids:
              lowerCamelCase   :      Union[str, Any]			 =					make_eval_dict(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,qid_list=_SCREAMING_SNAKE_CASE    )
              merge_eval(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,"NoAns"    )
       if OPTS.na_prob_file:
              find_all_best_thresh(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE    )
       if OPTS.na_prob_file and OPTS.out_image_dir:
              run_precision_recall_analysis(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,OPTS.out_image_dir    )
              histogram_na_prob(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,OPTS.out_image_dir ,"hasAns"    )
              histogram_na_prob(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,OPTS.out_image_dir ,"noAns"    )
       if OPTS.out_file:
              with open(OPTS.out_file ,"w"    ) as f:
                     json.dump(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE    )
       else:
              print(json.dumps(_SCREAMING_SNAKE_CASE ,indent=2    )    )


if __name__ == "__main__":
       SCREAMING_SNAKE_CASE__       :      List[str]  =   parse_args()
       if OPTS.out_image_dir:
              import matplotlib

              matplotlib.use('Agg')
              import matplotlib.pyplot as plt
       main()












from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


SCREAMING_SNAKE_CASE__       :      Optional[Any]  =   {'configuration_mmbt': ['MMBTConfig']}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       SCREAMING_SNAKE_CASE__       :      List[Any]  =   ['MMBTForClassification', 'MMBTModel', 'ModalEmbeddings']


if TYPE_CHECKING:
       from .configuration_mmbt import MMBTConfig

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_mmbt import MMBTForClassification, MMBTModel, ModalEmbeddings

else:
       import sys

       SCREAMING_SNAKE_CASE__       :      int  =   _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)






