





import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowercase      :       Union[str, Any]									=  re.compile(r"""\b(a|an|the)\b""", re.UNICODE)

lowercase      :       Union[str, Any]									=  None



def A_	(		)  ->		Dict:
       a__						:	Dict =				argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.'				)
       parser.add_argument('data_file'		,      metavar='data.json'		,      help='Input data JSON file.'				)
       parser.add_argument('pred_file'		,      metavar='pred.json'		,      help='Model predictions.'				)
       parser.add_argument(
           '--out-file'		,      '-o'		,      metavar='eval.json'		,      help='Write accuracy metrics to file (default is stdout).'				)
       parser.add_argument(
           '--na-prob-file'		,      '-n'		,      metavar='na_prob.json'		,      help='Model estimates of probability of no answer.'				)
       parser.add_argument(
           '--na-prob-thresh'		,      '-t'		,      type=A__		,      default=1.0		,      help='Predict "" if no-answer probability exceeds this (default = 1.0).'		,      )
       parser.add_argument(
           '--out-image-dir'		,      '-p'		,      metavar='out_images'		,      default=A__		,      help='Save precision-recall curves to directory.'				)
       parser.add_argument('--verbose'		,      '-v'		,      action='store_true'				)
       if len(sys.argv				) == 1:
              parser.print_help()
              sys.exit(1				)
       return parser.parse_args()



def A_	(		A__				)  ->		int:
       a__						:	Any =				{}
       for article in dataset:
              for p in article["paragraphs"]:
                     for qa in p["qas"]:
                            a__						:	Optional[int] =				bool(qa['answers']['text']				)
       return qid_to_has_ans



def A_	(		A__				)  ->		List[Any]:

       def remove_articles(A__				):
              return ARTICLES_REGEX.sub(' '		,      A__				)

       def white_space_fix(A__				):
              return " ".join(text.split()				)

       def remove_punc(A__				):
              a__						:	Optional[Any] =				set(string.punctuation				)
              return "".join(ch for ch in text if ch not in exclude				)

       def lower(A__				):
              return text.lower()

       return white_space_fix(remove_articles(remove_punc(lower(A__				)				)				)				)



def A_	(		A__				)  ->		Union[str, Any]:
       if not s:
              return []
       return normalize_answer(A__				).split()



def A_	(		A__		,      A__				)  ->		Optional[Any]:
       return int(normalize_answer(A__				) == normalize_answer(A__				)				)



def A_	(		A__		,      A__				)  ->		Any:
       a__						:	Tuple =				get_tokens(A__				)
       a__						:	Optional[int] =				get_tokens(A__				)
       a__						:	int =				collections.Counter(A__				) & collections.Counter(A__				)
       a__						:	Optional[Any] =				sum(common.values()				)
       if len(A__				) == 0 or len(A__				) == 0:
              # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
              return int(gold_toks == pred_toks				)
       if num_same == 0:
              return 0
       a__						:	int =				1.0 * num_same / len(A__				)
       a__						:	List[Any] =				1.0 * num_same / len(A__				)
       a__						:	Tuple =				(2 * precision * recall) / (precision + recall)
       return fa



def A_	(		A__		,      A__				)  ->		Any:
       a__						:	Tuple =				{}
       a__						:	int =				{}
       for article in dataset:
              for p in article["paragraphs"]:
                     for qa in p["qas"]:
                            a__						:	Optional[int] =				qa['id']
                            a__						:	Any =				[t for t in qa['answers']['text'] if normalize_answer(A__				)]
                            if not gold_answers:
                                   # For unanswerable questions, only correct answer is empty string
                                   a__						:	List[str] =				['']
                            if qid not in preds:
                                   print(F'Missing prediction for {qid}'				)
                                   continue
                            a__						:	Union[str, Any] =				preds[qid]
                            # Take max over all gold answers
                            a__						:	Tuple =				max(compute_exact(A__		,      A__				) for a in gold_answers				)
                            a__						:	List[Any] =				max(compute_fa(A__		,      A__				) for a in gold_answers				)
       return exact_scores, fa_scores



def A_	(		A__		,      A__		,      A__		,      A__				)  ->		Tuple:
       a__						:	List[Any] =				{}
       for qid, s in scores.items():
              a__						:	Tuple =				na_probs[qid] > na_prob_thresh
              if pred_na:
                     a__						:	str =				float(not qid_to_has_ans[qid]				)
              else:
                     a__						:	int =				s
       return new_scores



def A_	(		A__		,      A__		,      A__=None				)  ->		List[Any]:
       if not qid_list:
              a__						:	str =				len(A__				)
              return collections.OrderedDict(
                  [
                      ('exact', 1_00.0 * sum(exact_scores.values()				) / total),
                      ('f1', 1_00.0 * sum(fa_scores.values()				) / total),
                      ('total', total),
                  ]				)
       else:
              a__						:	int =				len(A__				)
              return collections.OrderedDict(
                  [
                      ('exact', 1_00.0 * sum(exact_scores[k] for k in qid_list				) / total),
                      ('f1', 1_00.0 * sum(fa_scores[k] for k in qid_list				) / total),
                      ('total', total),
                  ]				)



def A_	(		A__		,      A__		,      A__				)  ->		Optional[int]:
       for k in new_eval:
              a__						:	Optional[int] =				new_eval[k]



def A_	(		A__		,      A__		,      A__		,      A__				)  ->		Union[str, Any]:
       plt.step(A__		,      A__		,      color='b'		,      alpha=0.2		,      where='post'				)
       plt.fill_between(A__		,      A__		,      step='post'		,      alpha=0.2		,      color='b'				)
       plt.xlabel('Recall'				)
       plt.ylabel('Precision'				)
       plt.xlim([0.0, 1.05]				)
       plt.ylim([0.0, 1.05]				)
       plt.title(A__				)
       plt.savefig(A__				)
       plt.clf()



def A_	(		A__		,      A__		,      A__		,      A__		,      A__=None		,      A__=None				)  ->		Any:
       a__						:	str =				sorted(A__		,      key=lambda A__				: na_probs[k]				)
       a__						:	Tuple =				0.0
       a__						:	List[str] =				1.0
       a__						:	Optional[int] =				0.0
       a__						:	Any =				[1.0]
       a__						:	Optional[int] =				[0.0]
       a__						:	Tuple =				0.0
       for i, qid in enumerate(A__				):
              if qid_to_has_ans[qid]:
                     true_pos += scores[qid]
              a__						:	Union[str, Any] =				true_pos / float(i + 1				)
              a__						:	List[Any] =				true_pos / float(A__				)
              if i == len(A__				) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
                     # i.e., if we can put a threshold after this point
                     avg_prec += cur_p * (cur_r - recalls[-1])
                     precisions.append(A__				)
                     recalls.append(A__				)
       if out_image:
              plot_pr_curve(A__		,      A__		,      A__		,      A__				)
       return {"ap": 1_00.0 * avg_prec}



def A_	(		A__		,      A__		,      A__		,      A__		,      A__		,      A__				)  ->		str:
       if out_image_dir and not os.path.exists(A__				):
              os.makedirs(A__				)
       a__						:	List[str] =				sum(1 for v in qid_to_has_ans.values() if v				)
       if num_true_pos == 0:
              return
       a__						:	Optional[int] =				make_precision_recall_eval(
           A__		,      A__		,      A__		,      A__		,      out_image=os.path.join(A__		,      'pr_exact.png'				)		,      title='Precision-Recall curve for Exact Match score'		,      )
       a__						:	Optional[int] =				make_precision_recall_eval(
           A__		,      A__		,      A__		,      A__		,      out_image=os.path.join(A__		,      'pr_f1.png'				)		,      title='Precision-Recall curve for F1 score'		,      )
       a__						:	int =				{k: float(A__				) for k, v in qid_to_has_ans.items()}
       a__						:	Optional[Any] =				make_precision_recall_eval(
           A__		,      A__		,      A__		,      A__		,      out_image=os.path.join(A__		,      'pr_oracle.png'				)		,      title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'		,      )
       merge_eval(A__		,      A__		,      'pr_exact'				)
       merge_eval(A__		,      A__		,      'pr_f1'				)
       merge_eval(A__		,      A__		,      'pr_oracle'				)



def A_	(		A__		,      A__		,      A__		,      A__				)  ->		List[Any]:
       if not qid_list:
              return
       a__						:	List[str] =				[na_probs[k] for k in qid_list]
       a__						:	Dict =				np.ones_like(A__				) / float(len(A__				)				)
       plt.hist(A__		,      weights=A__		,      bins=20		,      range=(0.0, 1.0)				)
       plt.xlabel('Model probability of no-answer'				)
       plt.ylabel('Proportion of dataset'				)
       plt.title(F'Histogram of no-answer probability: {name}'				)
       plt.savefig(os.path.join(A__		,      F'na_prob_hist_{name}.png'				)				)
       plt.clf()



def A_	(		A__		,      A__		,      A__		,      A__				)  ->		Optional[int]:
       a__						:	Any =				sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]				)
       a__						:	List[str] =				num_no_ans
       a__						:	Tuple =				cur_score
       a__						:	Tuple =				0.0
       a__						:	str =				sorted(A__		,      key=lambda A__				: na_probs[k]				)
       for i, qid in enumerate(A__				):
              if qid not in scores:
                     continue
              if qid_to_has_ans[qid]:
                     a__						:	str =				scores[qid]
              else:
                     if preds[qid]:
                            a__						:	int =				-1
                     else:
                            a__						:	Tuple =				0
              cur_score += diff
              if cur_score > best_score:
                     a__						:	Dict =				cur_score
                     a__						:	Tuple =				na_probs[qid]
       return 1_00.0 * best_score / len(A__				), best_thresh



def A_	(		A__		,      A__		,      A__		,      A__		,      A__		,      A__				)  ->		Optional[Any]:
       a__ ,							a__						:	str =				find_best_thresh(A__		,      A__		,      A__		,      A__				)
       a__ ,							a__						:	Tuple =				find_best_thresh(A__		,      A__		,      A__		,      A__				)
       a__						:	Optional[Any] =				best_exact
       a__						:	Optional[Any] =				exact_thresh
       a__						:	Tuple =				best_fa
       a__						:	int =				fa_thresh



def A_	(		)  ->		Union[str, Any]:
       with open(OPTS.data_file				) as f:
              a__						:	Optional[int] =				json.load(A__				)
              a__						:	Any =				dataset_json['data']
       with open(OPTS.pred_file				) as f:
              a__						:	Optional[Any] =				json.load(A__				)
       if OPTS.na_prob_file:
              with open(OPTS.na_prob_file				) as f:
                     a__						:	int =				json.load(A__				)
       else:
              a__						:	Union[str, Any] =				{k: 0.0 for k in preds}
       a__						:	str =				make_qid_to_has_ans(A__				)  # maps qid to True/False
       a__						:	int =				[k for k, v in qid_to_has_ans.items() if v]
       a__						:	Tuple =				[k for k, v in qid_to_has_ans.items() if not v]
       a__ ,							a__						:	str =				get_raw_scores(A__		,      A__				)
       a__						:	str =				apply_no_ans_threshold(A__		,      A__		,      A__		,      OPTS.na_prob_thresh				)
       a__						:	Union[str, Any] =				apply_no_ans_threshold(A__		,      A__		,      A__		,      OPTS.na_prob_thresh				)
       a__						:	List[Any] =				make_eval_dict(A__		,      A__				)
       if has_ans_qids:
              a__						:	str =				make_eval_dict(A__		,      A__		,      qid_list=A__				)
              merge_eval(A__		,      A__		,      'HasAns'				)
       if no_ans_qids:
              a__						:	int =				make_eval_dict(A__		,      A__		,      qid_list=A__				)
              merge_eval(A__		,      A__		,      'NoAns'				)
       if OPTS.na_prob_file:
              find_all_best_thresh(A__		,      A__		,      A__		,      A__		,      A__		,      A__				)
       if OPTS.na_prob_file and OPTS.out_image_dir:
              run_precision_recall_analysis(A__		,      A__		,      A__		,      A__		,      A__		,      OPTS.out_image_dir				)
              histogram_na_prob(A__		,      A__		,      OPTS.out_image_dir		,      'hasAns'				)
              histogram_na_prob(A__		,      A__		,      OPTS.out_image_dir		,      'noAns'				)
       if OPTS.out_file:
              with open(OPTS.out_file		,      'w'				) as f:
                     json.dump(A__		,      A__				)
       else:
              print(json.dumps(A__		,      indent=2				)				)


if __name__ == "__main__":
    lowercase      :       List[Any]									=  parse_args()
    if OPTS.out_image_dir:
        import matplotlib

        matplotlib.use("""Agg""")
        import matplotlib.pyplot as plt
    main()









from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__lowerCamelCase       :     List[Any]   =     logging.get_logger(__name__)

__lowerCamelCase       :     str   =     {
    """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/config.json""",
    """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/config.json""",
    """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/config.json""",
    """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/config.json""",
    """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/config.json""",
    """roberta-large-openai-detector""": """https://huggingface.co/roberta-large-openai-detector/resolve/main/config.json""",
}



class 					A__   (       __snake_case			):
      _UpperCAmelCase	:Union[str, Any]						    =   'roberta'





      def __init__( self       ,							A_=5_0265       ,							A_=768       ,							A_=12       ,							A_=12       ,							A_=3072       ,							A_="gelu"       ,							A_=0.1       ,							A_=0.1       ,							A_=512       ,							A_=2       ,							A_=0.02       ,							A_=1e-12       ,							A_=1       ,							A_=0       ,							A_=2       ,							A_="absolute"       ,							A_=True       ,							A_=None       ,							**A_       ,							):
       '''simple docstring'''




       super().__init__(pad_token_id=A_       ,							bos_token_id=A_       ,							eos_token_id=A_       ,							**A_   )

       UpperCamelCase	:					Optional[int]       	=   vocab_size
       UpperCamelCase	:					Dict       	=   hidden_size
       UpperCamelCase	:					str       	=   num_hidden_layers
       UpperCamelCase	:					Any       	=   num_attention_heads
       UpperCamelCase	:					List[str]       	=   hidden_act
       UpperCamelCase	:					Optional[Any]       	=   intermediate_size
       UpperCamelCase	:					Tuple       	=   hidden_dropout_prob
       UpperCamelCase	:					Tuple       	=   attention_probs_dropout_prob
       UpperCamelCase	:					Tuple       	=   max_position_embeddings
       UpperCamelCase	:					Any       	=   type_vocab_size
       UpperCamelCase	:					int       	=   initializer_range
       UpperCamelCase	:					str       	=   layer_norm_eps
       UpperCamelCase	:					Dict       	=   position_embedding_type
       UpperCamelCase	:					Any       	=   use_cache
       UpperCamelCase	:					Union[str, Any]       	=   classifier_dropout



class 					A__   (       __snake_case			):





      @property
      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       if self.task == "multiple-choice":
        UpperCamelCase	:					Optional[int]       	=   {0: "batch", 1: "choice", 2: "sequence"}
       else:
        UpperCamelCase	:					Optional[int]       	=   {0: "batch", 1: "sequence"}
       return OrderedDict(
           [
               ("input_ids", dynamic_axis),
               ("attention_mask", dynamic_axis),
           ]   )
