
"""simple docstring"""






import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


__UpperCAmelCase       =  re.compile(R'\b(a|an|the)\b', re.UNICODE)

__UpperCAmelCase       =  None






def 						_snake_case       (	)  ->				List[str]:


			'''simple docstring'''





			lowerCAmelCase_    :Optional[int]			     =  argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0."""		)
			parser.add_argument("""data_file"""      ,     metavar="""data.json"""      ,     help="""Input data JSON file."""		)
			parser.add_argument("""pred_file"""      ,     metavar="""pred.json"""      ,     help="""Model predictions."""		)
			parser.add_argument(
			    """--out-file"""      ,     """-o"""      ,     metavar="""eval.json"""      ,     help="""Write accuracy metrics to file (default is stdout)."""		)
			parser.add_argument(
			    """--na-prob-file"""      ,     """-n"""      ,     metavar="""na_prob.json"""      ,     help="""Model estimates of probability of no answer."""		)
			parser.add_argument(
			    """--na-prob-thresh"""      ,     """-t"""      ,     type=lowercase__      ,     default=1.0      ,     help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""      ,     )
			parser.add_argument(
			    """--out-image-dir"""      ,     """-p"""      ,     metavar="""out_images"""      ,     default=lowercase__      ,     help="""Save precision-recall curves to directory."""		)
			parser.add_argument("""--verbose"""      ,     """-v"""      ,     action="""store_true"""		)
			if len(sys.argv		) == 1:
						parser.print_help()
						sys.exit(1		)
			return parser.parse_args()






def 						_snake_case       (	lowercase__		:  int		)  ->				Optional[Any]:


			'''simple docstring'''





			lowerCAmelCase_    :Tuple			     =  {}
			for article in dataset:
						for p in article["paragraphs"]:
									for qa in p["qas"]:
												lowerCAmelCase_    :Any			     =  bool(qa["""answers"""]["""text"""]		)
			return qid_to_has_ans






def 						_snake_case       (	lowercase__		:  Optional[Any]		)  ->				Any:


			'''simple docstring'''





			def remove_articles(lowercase__		:  int		):
						return ARTICLES_REGEX.sub(""" """      ,     lowercase__		)

			def white_space_fix(lowercase__		:  List[str]		):
						return " ".join(text.split()		)

			def remove_punc(lowercase__		:  Union[str, Any]		):
						lowerCAmelCase_    :List[Any]			     =  set(string.punctuation		)
						return "".join(ch for ch in text if ch not in exclude		)

			def lower(lowercase__		:  List[str]		):
						return text.lower()

			return white_space_fix(remove_articles(remove_punc(lower(lowercase__		)		)		)		)






def 						_snake_case       (	lowercase__		:  Optional[Any]		)  ->				Dict:


			'''simple docstring'''





			if not s:
						return []
			return normalize_answer(lowercase__		).split()






def 						_snake_case       (	lowercase__		:  Any      ,     lowercase__		:  Any		)  ->				Optional[int]:


			'''simple docstring'''





			return int(normalize_answer(lowercase__		) == normalize_answer(lowercase__		)		)






def 						_snake_case       (	lowercase__		:  List[str]      ,     lowercase__		:  int		)  ->				Optional[int]:


			'''simple docstring'''





			lowerCAmelCase_    :Any			     =  get_tokens(lowercase__		)
			lowerCAmelCase_    :str			     =  get_tokens(lowercase__		)
			lowerCAmelCase_    :Optional[Any]			     =  collections.Counter(lowercase__		) & collections.Counter(lowercase__		)
			lowerCAmelCase_    :Optional[int]			     =  sum(common.values()		)
			if len(lowercase__		) == 0 or len(lowercase__		) == 0:
						# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
						return int(gold_toks == pred_toks		)
			if num_same == 0:
						return 0
			lowerCAmelCase_    :Dict			     =  1.0 * num_same / len(lowercase__		)
			lowerCAmelCase_    :str			     =  1.0 * num_same / len(lowercase__		)
			lowerCAmelCase_    :List[Any]			     =  (2 * precision * recall) / (precision + recall)
			return fa






def 						_snake_case       (	lowercase__		:  Tuple      ,     lowercase__		:  str		)  ->				Optional[int]:


			'''simple docstring'''





			lowerCAmelCase_    :int			     =  {}
			lowerCAmelCase_    :Dict			     =  {}
			for article in dataset:
						for p in article["paragraphs"]:
									for qa in p["qas"]:
												lowerCAmelCase_    :Tuple			     =  qa["""id"""]
												lowerCAmelCase_    :Union[str, Any]			     =  [t for t in qa["""answers"""]["""text"""] if normalize_answer(lowercase__		)]
												if not gold_answers:
															# For unanswerable questions, only correct answer is empty string
															lowerCAmelCase_    :Any			     =  [""""""]
												if qid not in preds:
															print(f"""Missing prediction for {qid}"""		)
															continue
												lowerCAmelCase_    :List[str]			     =  preds[qid]
												# Take max over all gold answers
												lowerCAmelCase_    :Any			     =  max(compute_exact(lowercase__      ,     lowercase__		) for a in gold_answers		)
												lowerCAmelCase_    :List[str]			     =  max(compute_fa(lowercase__      ,     lowercase__		) for a in gold_answers		)
			return exact_scores, fa_scores






def 						_snake_case       (	lowercase__		:  Any      ,     lowercase__		:  Dict      ,     lowercase__		:  Tuple      ,     lowercase__		:  List[str]		)  ->				Optional[Any]:


			'''simple docstring'''





			lowerCAmelCase_    :Any			     =  {}
			for qid, s in scores.items():
						lowerCAmelCase_    :str			     =  na_probs[qid] > na_prob_thresh
						if pred_na:
									lowerCAmelCase_    :List[str]			     =  float(not qid_to_has_ans[qid]		)
						else:
									lowerCAmelCase_    :Optional[Any]			     =  s
			return new_scores






def 						_snake_case       (	lowercase__		:  List[str]      ,     lowercase__		:  Dict      ,     lowercase__		:  str=None		)  ->				Dict:


			'''simple docstring'''





			if not qid_list:
						lowerCAmelCase_    :Dict			     =  len(lowercase__		)
						return collections.OrderedDict(
						    [
						        ("""exact""", 100.0 * sum(exact_scores.values()		) / total),
						        ("""f1""", 100.0 * sum(fa_scores.values()		) / total),
						        ("""total""", total),
						    ]		)
			else:
						lowerCAmelCase_    :int			     =  len(lowercase__		)
						return collections.OrderedDict(
						    [
						        ("""exact""", 100.0 * sum(exact_scores[k] for k in qid_list		) / total),
						        ("""f1""", 100.0 * sum(fa_scores[k] for k in qid_list		) / total),
						        ("""total""", total),
						    ]		)






def 						_snake_case       (	lowercase__		:  Dict      ,     lowercase__		:  List[Any]      ,     lowercase__		:  List[str]		)  ->				Tuple:


			'''simple docstring'''





			for k in new_eval:
						lowerCAmelCase_    :int			     =  new_eval[k]






def 						_snake_case       (	lowercase__		:  Optional[Any]      ,     lowercase__		:  Any      ,     lowercase__		:  str      ,     lowercase__		:  str		)  ->				Dict:


			'''simple docstring'''





			plt.step(lowercase__      ,     lowercase__      ,     color="""b"""      ,     alpha=0.2      ,     where="""post"""		)
			plt.fill_between(lowercase__      ,     lowercase__      ,     step="""post"""      ,     alpha=0.2      ,     color="""b"""		)
			plt.xlabel("""Recall"""		)
			plt.ylabel("""Precision"""		)
			plt.xlim([0.0, 1.05]		)
			plt.ylim([0.0, 1.05]		)
			plt.title(lowercase__		)
			plt.savefig(lowercase__		)
			plt.clf()






def 						_snake_case       (	lowercase__		:  int      ,     lowercase__		:  int      ,     lowercase__		:  Tuple      ,     lowercase__		:  List[str]      ,     lowercase__		:  Union[str, Any]=None      ,     lowercase__		:  List[Any]=None		)  ->				int:


			'''simple docstring'''





			lowerCAmelCase_    :int			     =  sorted(lowercase__      ,     key=lambda lowercase__		: na_probs[k]		)
			lowerCAmelCase_    :Tuple			     =  0.0
			lowerCAmelCase_    :List[str]			     =  1.0
			lowerCAmelCase_    :Optional[Any]			     =  0.0
			lowerCAmelCase_    :List[str]			     =  [1.0]
			lowerCAmelCase_    :Dict			     =  [0.0]
			lowerCAmelCase_    :str			     =  0.0
			for i, qid in enumerate(lowercase__		):
						if qid_to_has_ans[qid]:
									true_pos += scores[qid]
						lowerCAmelCase_    :List[Any]			     =  true_pos / float(i + 1		)
						lowerCAmelCase_    :List[Any]			     =  true_pos / float(lowercase__		)
						if i == len(lowercase__		) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
									# i.e., if we can put a threshold after this point
									avg_prec += cur_p * (cur_r - recalls[-1])
									precisions.append(lowercase__		)
									recalls.append(lowercase__		)
			if out_image:
						plot_pr_curve(lowercase__      ,     lowercase__      ,     lowercase__      ,     lowercase__		)
			return {"ap": 100.0 * avg_prec}






def 						_snake_case       (	lowercase__		:  str      ,     lowercase__		:  Tuple      ,     lowercase__		:  int      ,     lowercase__		:  Any      ,     lowercase__		:  str      ,     lowercase__		:  int		)  ->				Union[str, Any]:


			'''simple docstring'''





			if out_image_dir and not os.path.exists(lowercase__		):
						os.makedirs(lowercase__		)
			lowerCAmelCase_    :Dict			     =  sum(1 for v in qid_to_has_ans.values() if v		)
			if num_true_pos == 0:
						return
			lowerCAmelCase_    :Optional[int]			     =  make_precision_recall_eval(
			    lowercase__      ,     lowercase__      ,     lowercase__      ,     lowercase__      ,     out_image=os.path.join(lowercase__      ,     """pr_exact.png"""		)      ,     title="""Precision-Recall curve for Exact Match score"""      ,     )
			lowerCAmelCase_    :int			     =  make_precision_recall_eval(
			    lowercase__      ,     lowercase__      ,     lowercase__      ,     lowercase__      ,     out_image=os.path.join(lowercase__      ,     """pr_f1.png"""		)      ,     title="""Precision-Recall curve for F1 score"""      ,     )
			lowerCAmelCase_    :List[str]			     =  {k: float(lowercase__		) for k, v in qid_to_has_ans.items()}
			lowerCAmelCase_    :str			     =  make_precision_recall_eval(
			    lowercase__      ,     lowercase__      ,     lowercase__      ,     lowercase__      ,     out_image=os.path.join(lowercase__      ,     """pr_oracle.png"""		)      ,     title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""      ,     )
			merge_eval(lowercase__      ,     lowercase__      ,     """pr_exact"""		)
			merge_eval(lowercase__      ,     lowercase__      ,     """pr_f1"""		)
			merge_eval(lowercase__      ,     lowercase__      ,     """pr_oracle"""		)






def 						_snake_case       (	lowercase__		:  Tuple      ,     lowercase__		:  List[Any]      ,     lowercase__		:  Dict      ,     lowercase__		:  int		)  ->				List[Any]:


			'''simple docstring'''





			if not qid_list:
						return
			lowerCAmelCase_    :str			     =  [na_probs[k] for k in qid_list]
			lowerCAmelCase_    :Tuple			     =  np.ones_like(lowercase__		) / float(len(lowercase__		)		)
			plt.hist(lowercase__      ,     weights=lowercase__      ,     bins=2_0      ,     range=(0.0, 1.0)		)
			plt.xlabel("""Model probability of no-answer"""		)
			plt.ylabel("""Proportion of dataset"""		)
			plt.title(f"""Histogram of no-answer probability: {name}"""		)
			plt.savefig(os.path.join(lowercase__      ,     f"""na_prob_hist_{name}.png"""		)		)
			plt.clf()






def 						_snake_case       (	lowercase__		:  Tuple      ,     lowercase__		:  Optional[int]      ,     lowercase__		:  Tuple      ,     lowercase__		:  Optional[Any]		)  ->				Union[str, Any]:


			'''simple docstring'''





			lowerCAmelCase_    :int			     =  sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]		)
			lowerCAmelCase_    :List[Any]			     =  num_no_ans
			lowerCAmelCase_    :int			     =  cur_score
			lowerCAmelCase_    :Optional[Any]			     =  0.0
			lowerCAmelCase_    :Union[str, Any]			     =  sorted(lowercase__      ,     key=lambda lowercase__		: na_probs[k]		)
			for i, qid in enumerate(lowercase__		):
						if qid not in scores:
									continue
						if qid_to_has_ans[qid]:
									lowerCAmelCase_    :Optional[int]			     =  scores[qid]
						else:
									if preds[qid]:
												lowerCAmelCase_    :Optional[int]			     =  -1
									else:
												lowerCAmelCase_    :Union[str, Any]			     =  0
						cur_score += diff
						if cur_score > best_score:
									lowerCAmelCase_    :Dict			     =  cur_score
									lowerCAmelCase_    :Tuple			     =  na_probs[qid]
			return 100.0 * best_score / len(lowercase__		), best_thresh






def 						_snake_case       (	lowercase__		:  Dict      ,     lowercase__		:  List[str]      ,     lowercase__		:  Optional[int]      ,     lowercase__		:  Optional[Any]      ,     lowercase__		:  Any      ,     lowercase__		:  Union[str, Any]		)  ->				List[Any]:


			'''simple docstring'''





			lowerCAmelCase_     ,       lowerCAmelCase_    :Any			     =  find_best_thresh(lowercase__      ,     lowercase__      ,     lowercase__      ,     lowercase__		)
			lowerCAmelCase_     ,       lowerCAmelCase_    :Union[str, Any]			     =  find_best_thresh(lowercase__      ,     lowercase__      ,     lowercase__      ,     lowercase__		)
			lowerCAmelCase_    :Optional[Any]			     =  best_exact
			lowerCAmelCase_    :List[Any]			     =  exact_thresh
			lowerCAmelCase_    :Tuple			     =  best_fa
			lowerCAmelCase_    :str			     =  fa_thresh






def 						_snake_case       (	)  ->				Any:


			'''simple docstring'''





			with open(OPTS.data_file		) as f:
						lowerCAmelCase_    :Union[str, Any]			     =  json.load(lowercase__		)
						lowerCAmelCase_    :Optional[Any]			     =  dataset_json["""data"""]
			with open(OPTS.pred_file		) as f:
						lowerCAmelCase_    :Dict			     =  json.load(lowercase__		)
			if OPTS.na_prob_file:
						with open(OPTS.na_prob_file		) as f:
									lowerCAmelCase_    :Tuple			     =  json.load(lowercase__		)
			else:
						lowerCAmelCase_    :str			     =  {k: 0.0 for k in preds}
			lowerCAmelCase_    :List[str]			     =  make_qid_to_has_ans(lowercase__		)  # maps qid to True/False
			lowerCAmelCase_    :List[str]			     =  [k for k, v in qid_to_has_ans.items() if v]
			lowerCAmelCase_    :int			     =  [k for k, v in qid_to_has_ans.items() if not v]
			lowerCAmelCase_     ,       lowerCAmelCase_    :str			     =  get_raw_scores(lowercase__      ,     lowercase__		)
			lowerCAmelCase_    :Optional[Any]			     =  apply_no_ans_threshold(lowercase__      ,     lowercase__      ,     lowercase__      ,     OPTS.na_prob_thresh		)
			lowerCAmelCase_    :Union[str, Any]			     =  apply_no_ans_threshold(lowercase__      ,     lowercase__      ,     lowercase__      ,     OPTS.na_prob_thresh		)
			lowerCAmelCase_    :Any			     =  make_eval_dict(lowercase__      ,     lowercase__		)
			if has_ans_qids:
						lowerCAmelCase_    :Dict			     =  make_eval_dict(lowercase__      ,     lowercase__      ,     qid_list=lowercase__		)
						merge_eval(lowercase__      ,     lowercase__      ,     """HasAns"""		)
			if no_ans_qids:
						lowerCAmelCase_    :List[Any]			     =  make_eval_dict(lowercase__      ,     lowercase__      ,     qid_list=lowercase__		)
						merge_eval(lowercase__      ,     lowercase__      ,     """NoAns"""		)
			if OPTS.na_prob_file:
						find_all_best_thresh(lowercase__      ,     lowercase__      ,     lowercase__      ,     lowercase__      ,     lowercase__      ,     lowercase__		)
			if OPTS.na_prob_file and OPTS.out_image_dir:
						run_precision_recall_analysis(lowercase__      ,     lowercase__      ,     lowercase__      ,     lowercase__      ,     lowercase__      ,     OPTS.out_image_dir		)
						histogram_na_prob(lowercase__      ,     lowercase__      ,     OPTS.out_image_dir      ,     """hasAns"""		)
						histogram_na_prob(lowercase__      ,     lowercase__      ,     OPTS.out_image_dir      ,     """noAns"""		)
			if OPTS.out_file:
						with open(OPTS.out_file      ,     """w"""		) as f:
									json.dump(lowercase__      ,     lowercase__		)
			else:
						print(json.dumps(lowercase__      ,     indent=2		)		)


if __name__ == "__main__":
				__UpperCAmelCase       =  parse_args()
				if OPTS.out_image_dir:
								import matplotlib

								matplotlib.use('Agg')
								import matplotlib.pyplot as plt
				main()










import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding
class  SCREAMING_SNAKE_CASE__  (				lowercase__			):
      snake_case__				:     Tuple        =						['''image_processor''', '''tokenizer''']
      snake_case__				:     Union[str, Any]        =						'''CLIPImageProcessor'''
      snake_case__				:     Dict        =						('''CLIPTokenizer''', '''CLIPTokenizerFast''')
      def __init__(       self				:       Optional[int]     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]=None     ,							SCREAMING_SNAKE_CASE__				:       List[str]=None     ,							**SCREAMING_SNAKE_CASE__				:       int     )		->			Any:
             a_					:		List[Any]							=  None
             if "feature_extractor" in kwargs:
                    warnings.warn(
                        'The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'
                        ' instead.'     ,							SCREAMING_SNAKE_CASE__     ,							)
                    a_					:		Tuple							=  kwargs.pop('feature_extractor'     )

             a_					:		Tuple							=  image_processor if image_processor is not None else feature_extractor
             if image_processor is None:
                    raise ValueError('You need to specify an `image_processor`.'     )
             if tokenizer is None:
                    raise ValueError('You need to specify a `tokenizer`.'     )

             super().__init__(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     )
      def __call__(       self				:       List[Any]     ,							SCREAMING_SNAKE_CASE__				:       Optional[Any]=None     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]=None     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]=None     ,							**SCREAMING_SNAKE_CASE__				:       str     )		->			Optional[Any]:

             if text is None and images is None:
                    raise ValueError('You have to specify either text or images. Both cannot be none.'     )

             if text is not None:
                    a_					:		List[str]							=  self.tokenizer(SCREAMING_SNAKE_CASE__     ,							return_tensors=SCREAMING_SNAKE_CASE__     ,							**SCREAMING_SNAKE_CASE__     )

             if images is not None:
                    a_					:		Dict							=  self.image_processor(SCREAMING_SNAKE_CASE__     ,							return_tensors=SCREAMING_SNAKE_CASE__     ,							**SCREAMING_SNAKE_CASE__     )

             if text is not None and images is not None:
                    a_					:		Dict							=  image_features.pixel_values
                    return encoding
             elif text is not None:
                    return encoding
             else:
                    return BatchEncoding(data=dict(**SCREAMING_SNAKE_CASE__     )     ,							tensor_type=SCREAMING_SNAKE_CASE__     )
      def        SCREAMING_SNAKE_CASE							(       self				:       Any     ,							*SCREAMING_SNAKE_CASE__				:       List[Any]     ,							**SCREAMING_SNAKE_CASE__				:       Optional[int]     )		->			List[Any]:
             return self.tokenizer.batch_decode(*SCREAMING_SNAKE_CASE__     ,							**SCREAMING_SNAKE_CASE__     )
      def        SCREAMING_SNAKE_CASE							(       self				:       Optional[Any]     ,							*SCREAMING_SNAKE_CASE__				:       List[Any]     ,							**SCREAMING_SNAKE_CASE__				:       Dict     )		->			Optional[Any]:
             return self.tokenizer.decode(*SCREAMING_SNAKE_CASE__     ,							**SCREAMING_SNAKE_CASE__     )
      @property
      def        SCREAMING_SNAKE_CASE							(       self				:       List[Any]     )		->			Any:
             a_					:		str							=  self.tokenizer.model_input_names
             a_					:		Tuple							=  self.image_processor.model_input_names
             return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names     )     )
      @property
      def        SCREAMING_SNAKE_CASE							(       self				:       str     )		->			str:
             warnings.warn(
                 '`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'     ,							SCREAMING_SNAKE_CASE__     ,							)
             return self.image_processor_class

      @property
      def        SCREAMING_SNAKE_CASE							(       self				:       str     )		->			Tuple:
             warnings.warn(
                 '`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'     ,							SCREAMING_SNAKE_CASE__     ,							)
             return self.image_processor


