"""simple docstring"""





import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


_UpperCamelCase   :						Any      =    re.compile(r"\b(a|an|the)\b", re.UNICODE)

_UpperCamelCase   :						Union[str, Any]      =    None





def 				a_     (      ):





       '''simple docstring'''

       lowercase__			:		Optional[int]       =	argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.'     )
       parser.add_argument('data_file'   , metavar='data.json'   , help='Input data JSON file.'     )
       parser.add_argument('pred_file'   , metavar='pred.json'   , help='Model predictions.'     )
       parser.add_argument(
           '--out-file'   , '-o'   , metavar='eval.json'   , help='Write accuracy metrics to file (default is stdout).'     )
       parser.add_argument(
           '--na-prob-file'   , '-n'   , metavar='na_prob.json'   , help='Model estimates of probability of no answer.'     )
       parser.add_argument(
           '--na-prob-thresh'   , '-t'   , type=_lowerCAmelCase   , default=1.0   , help='Predict "" if no-answer probability exceeds this (default = 1.0).'   , )
       parser.add_argument(
           '--out-image-dir'   , '-p'   , metavar='out_images'   , default=_lowerCAmelCase   , help='Save precision-recall curves to directory.'     )
       parser.add_argument('--verbose'   , '-v'   , action='store_true'     )
       if len(sys.argv     ) == 1:
              parser.print_help()
              sys.exit(1     )
       return parser.parse_args()





def 				a_     (      _lowerCAmelCase  : Optional[Any]     ):





       '''simple docstring'''

       lowercase__			:		Tuple       =	{}
       for article in dataset:
              for p in article["paragraphs"]:
                     for qa in p["qas"]:
                            lowercase__			:		Optional[int]       =	bool(qa['answers']['text']     )
       return qid_to_has_ans





def 				a_     (      _lowerCAmelCase  : Any     ):





       '''simple docstring'''

       def remove_articles(_lowerCAmelCase  : int     ):
              return ARTICLES_REGEX.sub(' '   , _lowerCAmelCase     )

       def white_space_fix(_lowerCAmelCase  : str     ):
              return " ".join(text.split()     )

       def remove_punc(_lowerCAmelCase  : List[Any]     ):
              lowercase__			:		int       =	set(string.punctuation     )
              return "".join(ch for ch in text if ch not in exclude     )

       def lower(_lowerCAmelCase  : List[str]     ):
              return text.lower()

       return white_space_fix(remove_articles(remove_punc(lower(_lowerCAmelCase     )     )     )     )





def 				a_     (      _lowerCAmelCase  : Union[str, Any]     ):





       '''simple docstring'''

       if not s:
              return []
       return normalize_answer(_lowerCAmelCase     ).split()





def 				a_     (      _lowerCAmelCase  : Optional[Any]   , _lowerCAmelCase  : str     ):





       '''simple docstring'''

       return int(normalize_answer(_lowerCAmelCase     ) == normalize_answer(_lowerCAmelCase     )     )





def 				a_     (      _lowerCAmelCase  : Dict   , _lowerCAmelCase  : Dict     ):





       '''simple docstring'''

       lowercase__			:		Dict       =	get_tokens(_lowerCAmelCase     )
       lowercase__			:		List[str]       =	get_tokens(_lowerCAmelCase     )
       lowercase__			:		List[Any]       =	collections.Counter(_lowerCAmelCase     ) & collections.Counter(_lowerCAmelCase     )
       lowercase__			:		int       =	sum(common.values()     )
       if len(_lowerCAmelCase     ) == 0 or len(_lowerCAmelCase     ) == 0:
              # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
              return int(gold_toks == pred_toks     )
       if num_same == 0:
              return 0
       lowercase__			:		Any       =	1.0 * num_same / len(_lowerCAmelCase     )
       lowercase__			:		Dict       =	1.0 * num_same / len(_lowerCAmelCase     )
       lowercase__			:		Any       =	(2 * precision * recall) / (precision + recall)
       return fa





def 				a_     (      _lowerCAmelCase  : Tuple   , _lowerCAmelCase  : Optional[Any]     ):





       '''simple docstring'''

       lowercase__			:		Optional[int]       =	{}
       lowercase__			:		Union[str, Any]       =	{}
       for article in dataset:
              for p in article["paragraphs"]:
                     for qa in p["qas"]:
                            lowercase__			:		Any       =	qa['id']
                            lowercase__			:		Union[str, Any]       =	[t for t in qa['answers']['text'] if normalize_answer(_lowerCAmelCase     )]
                            if not gold_answers:
                                   # For unanswerable questions, only correct answer is empty string
                                   lowercase__			:		Dict       =	['']
                            if qid not in preds:
                                   print(f"""Missing prediction for {qid}"""     )
                                   continue
                            lowercase__			:		Optional[int]       =	preds[qid]
                            # Take max over all gold answers
                            lowercase__			:		int       =	max(compute_exact(_lowerCAmelCase   , _lowerCAmelCase     ) for a in gold_answers     )
                            lowercase__			:		Optional[Any]       =	max(compute_fa(_lowerCAmelCase   , _lowerCAmelCase     ) for a in gold_answers     )
       return exact_scores, fa_scores





def 				a_     (      _lowerCAmelCase  : List[str]   , _lowerCAmelCase  : Tuple   , _lowerCAmelCase  : Tuple   , _lowerCAmelCase  : List[str]     ):





       '''simple docstring'''

       lowercase__			:		str       =	{}
       for qid, s in scores.items():
              lowercase__			:		int       =	na_probs[qid] > na_prob_thresh
              if pred_na:
                     lowercase__			:		Optional[Any]       =	float(not qid_to_has_ans[qid]     )
              else:
                     lowercase__			:		Optional[Any]       =	s
       return new_scores





def 				a_     (      _lowerCAmelCase  : str   , _lowerCAmelCase  : List[Any]   , _lowerCAmelCase  : str=None     ):





       '''simple docstring'''

       if not qid_list:
              lowercase__			:		Optional[Any]       =	len(_lowerCAmelCase     )
              return collections.OrderedDict(
                  [
                      ('exact', 1_0_0.0 * sum(exact_scores.values()     ) / total),
                      ('f1', 1_0_0.0 * sum(fa_scores.values()     ) / total),
                      ('total', total),
                  ]     )
       else:
              lowercase__			:		Optional[Any]       =	len(_lowerCAmelCase     )
              return collections.OrderedDict(
                  [
                      ('exact', 1_0_0.0 * sum(exact_scores[k] for k in qid_list     ) / total),
                      ('f1', 1_0_0.0 * sum(fa_scores[k] for k in qid_list     ) / total),
                      ('total', total),
                  ]     )





def 				a_     (      _lowerCAmelCase  : str   , _lowerCAmelCase  : Optional[Any]   , _lowerCAmelCase  : Union[str, Any]     ):





       '''simple docstring'''

       for k in new_eval:
              lowercase__			:		int       =	new_eval[k]





def 				a_     (      _lowerCAmelCase  : str   , _lowerCAmelCase  : Optional[Any]   , _lowerCAmelCase  : int   , _lowerCAmelCase  : Tuple     ):





       '''simple docstring'''

       plt.step(_lowerCAmelCase   , _lowerCAmelCase   , color='b'   , alpha=0.2   , where='post'     )
       plt.fill_between(_lowerCAmelCase   , _lowerCAmelCase   , step='post'   , alpha=0.2   , color='b'     )
       plt.xlabel('Recall'     )
       plt.ylabel('Precision'     )
       plt.xlim([0.0, 1.0_5]     )
       plt.ylim([0.0, 1.0_5]     )
       plt.title(_lowerCAmelCase     )
       plt.savefig(_lowerCAmelCase     )
       plt.clf()





def 				a_     (      _lowerCAmelCase  : Union[str, Any]   , _lowerCAmelCase  : str   , _lowerCAmelCase  : int   , _lowerCAmelCase  : str   , _lowerCAmelCase  : Any=None   , _lowerCAmelCase  : List[str]=None     ):





       '''simple docstring'''

       lowercase__			:		Optional[int]       =	sorted(_lowerCAmelCase   , key=lambda _lowerCAmelCase     : na_probs[k]     )
       lowercase__			:		Tuple       =	0.0
       lowercase__			:		List[str]       =	1.0
       lowercase__			:		List[str]       =	0.0
       lowercase__			:		Union[str, Any]       =	[1.0]
       lowercase__			:		List[Any]       =	[0.0]
       lowercase__			:		Optional[int]       =	0.0
       for i, qid in enumerate(_lowerCAmelCase     ):
              if qid_to_has_ans[qid]:
                     true_pos += scores[qid]
              lowercase__			:		Tuple       =	true_pos / float(i + 1     )
              lowercase__			:		Union[str, Any]       =	true_pos / float(_lowerCAmelCase     )
              if i == len(_lowerCAmelCase     ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
                     # i.e., if we can put a threshold after this point
                     avg_prec += cur_p * (cur_r - recalls[-1])
                     precisions.append(_lowerCAmelCase     )
                     recalls.append(_lowerCAmelCase     )
       if out_image:
              plot_pr_curve(_lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase     )
       return {"ap": 1_0_0.0 * avg_prec}





def 				a_     (      _lowerCAmelCase  : Dict   , _lowerCAmelCase  : Tuple   , _lowerCAmelCase  : Tuple   , _lowerCAmelCase  : Tuple   , _lowerCAmelCase  : Optional[Any]   , _lowerCAmelCase  : Tuple     ):





       '''simple docstring'''

       if out_image_dir and not os.path.exists(_lowerCAmelCase     ):
              os.makedirs(_lowerCAmelCase     )
       lowercase__			:		List[str]       =	sum(1 for v in qid_to_has_ans.values() if v     )
       if num_true_pos == 0:
              return
       lowercase__			:		Dict       =	make_precision_recall_eval(
           _lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase   , out_image=os.path.join(_lowerCAmelCase   , 'pr_exact.png'     )   , title='Precision-Recall curve for Exact Match score'   , )
       lowercase__			:		Tuple       =	make_precision_recall_eval(
           _lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase   , out_image=os.path.join(_lowerCAmelCase   , 'pr_f1.png'     )   , title='Precision-Recall curve for F1 score'   , )
       lowercase__			:		List[Any]       =	{k: float(_lowerCAmelCase     ) for k, v in qid_to_has_ans.items()}
       lowercase__			:		Any       =	make_precision_recall_eval(
           _lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase   , out_image=os.path.join(_lowerCAmelCase   , 'pr_oracle.png'     )   , title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'   , )
       merge_eval(_lowerCAmelCase   , _lowerCAmelCase   , 'pr_exact'     )
       merge_eval(_lowerCAmelCase   , _lowerCAmelCase   , 'pr_f1'     )
       merge_eval(_lowerCAmelCase   , _lowerCAmelCase   , 'pr_oracle'     )





def 				a_     (      _lowerCAmelCase  : int   , _lowerCAmelCase  : Any   , _lowerCAmelCase  : Tuple   , _lowerCAmelCase  : Optional[Any]     ):





       '''simple docstring'''

       if not qid_list:
              return
       lowercase__			:		List[str]       =	[na_probs[k] for k in qid_list]
       lowercase__			:		Tuple       =	np.ones_like(_lowerCAmelCase     ) / float(len(_lowerCAmelCase     )     )
       plt.hist(_lowerCAmelCase   , weights=_lowerCAmelCase   , bins=20   , range=(0.0, 1.0)     )
       plt.xlabel('Model probability of no-answer'     )
       plt.ylabel('Proportion of dataset'     )
       plt.title(f"""Histogram of no-answer probability: {name}"""     )
       plt.savefig(os.path.join(_lowerCAmelCase   , f"""na_prob_hist_{name}.png"""     )     )
       plt.clf()





def 				a_     (      _lowerCAmelCase  : Optional[Any]   , _lowerCAmelCase  : Dict   , _lowerCAmelCase  : Any   , _lowerCAmelCase  : Union[str, Any]     ):





       '''simple docstring'''

       lowercase__			:		Tuple       =	sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]     )
       lowercase__			:		int       =	num_no_ans
       lowercase__			:		Optional[int]       =	cur_score
       lowercase__			:		Tuple       =	0.0
       lowercase__			:		Dict       =	sorted(_lowerCAmelCase   , key=lambda _lowerCAmelCase     : na_probs[k]     )
       for i, qid in enumerate(_lowerCAmelCase     ):
              if qid not in scores:
                     continue
              if qid_to_has_ans[qid]:
                     lowercase__			:		Optional[int]       =	scores[qid]
              else:
                     if preds[qid]:
                            lowercase__			:		List[Any]       =	-1
                     else:
                            lowercase__			:		Optional[int]       =	0
              cur_score += diff
              if cur_score > best_score:
                     lowercase__			:		Dict       =	cur_score
                     lowercase__			:		Optional[int]       =	na_probs[qid]
       return 1_0_0.0 * best_score / len(_lowerCAmelCase     ), best_thresh





def 				a_     (      _lowerCAmelCase  : Optional[int]   , _lowerCAmelCase  : Optional[int]   , _lowerCAmelCase  : Tuple   , _lowerCAmelCase  : Union[str, Any]   , _lowerCAmelCase  : Union[str, Any]   , _lowerCAmelCase  : str     ):





       '''simple docstring'''

       lowercase__    ,      lowercase__			:		List[Any]       =	find_best_thresh(_lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase     )
       lowercase__    ,      lowercase__			:		Dict       =	find_best_thresh(_lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase     )
       lowercase__			:		Any       =	best_exact
       lowercase__			:		Tuple       =	exact_thresh
       lowercase__			:		Optional[Any]       =	best_fa
       lowercase__			:		Any       =	fa_thresh





def 				a_     (      ):





       '''simple docstring'''

       with open(OPTS.data_file     ) as f:
              lowercase__			:		List[Any]       =	json.load(_lowerCAmelCase     )
              lowercase__			:		Union[str, Any]       =	dataset_json['data']
       with open(OPTS.pred_file     ) as f:
              lowercase__			:		str       =	json.load(_lowerCAmelCase     )
       if OPTS.na_prob_file:
              with open(OPTS.na_prob_file     ) as f:
                     lowercase__			:		Union[str, Any]       =	json.load(_lowerCAmelCase     )
       else:
              lowercase__			:		str       =	{k: 0.0 for k in preds}
       lowercase__			:		int       =	make_qid_to_has_ans(_lowerCAmelCase     )  # maps qid to True/False
       lowercase__			:		List[str]       =	[k for k, v in qid_to_has_ans.items() if v]
       lowercase__			:		Any       =	[k for k, v in qid_to_has_ans.items() if not v]
       lowercase__    ,      lowercase__			:		Any       =	get_raw_scores(_lowerCAmelCase   , _lowerCAmelCase     )
       lowercase__			:		Optional[Any]       =	apply_no_ans_threshold(_lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase   , OPTS.na_prob_thresh     )
       lowercase__			:		Union[str, Any]       =	apply_no_ans_threshold(_lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase   , OPTS.na_prob_thresh     )
       lowercase__			:		Tuple       =	make_eval_dict(_lowerCAmelCase   , _lowerCAmelCase     )
       if has_ans_qids:
              lowercase__			:		int       =	make_eval_dict(_lowerCAmelCase   , _lowerCAmelCase   , qid_list=_lowerCAmelCase     )
              merge_eval(_lowerCAmelCase   , _lowerCAmelCase   , 'HasAns'     )
       if no_ans_qids:
              lowercase__			:		Optional[Any]       =	make_eval_dict(_lowerCAmelCase   , _lowerCAmelCase   , qid_list=_lowerCAmelCase     )
              merge_eval(_lowerCAmelCase   , _lowerCAmelCase   , 'NoAns'     )
       if OPTS.na_prob_file:
              find_all_best_thresh(_lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase     )
       if OPTS.na_prob_file and OPTS.out_image_dir:
              run_precision_recall_analysis(_lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase   , OPTS.out_image_dir     )
              histogram_na_prob(_lowerCAmelCase   , _lowerCAmelCase   , OPTS.out_image_dir   , 'hasAns'     )
              histogram_na_prob(_lowerCAmelCase   , _lowerCAmelCase   , OPTS.out_image_dir   , 'noAns'     )
       if OPTS.out_file:
              with open(OPTS.out_file   , 'w'     ) as f:
                     json.dump(_lowerCAmelCase   , _lowerCAmelCase     )
       else:
              print(json.dumps(_lowerCAmelCase   , indent=2     )     )


if __name__ == "__main__":
       _UpperCamelCase   :						Optional[int]      =    parse_args()
       if OPTS.out_image_dir:
              import matplotlib

              matplotlib.use("Agg")
              import matplotlib.pyplot as plt
       main()



import unittest

import numpy as np

from diffusers import LMSDiscreteScheduler, OnnxStableDiffusionInpaintPipeline
from diffusers.utils.testing_utils import (
    is_onnx_available,
    load_image,
    nightly,
    require_onnxruntime,
    require_torch_gpu,
)

from ..test_pipelines_onnx_common import OnnxPipelineTesterMixin


if is_onnx_available():
					import onnxruntime as ort







class 			__A( a  ,   unittest.TestCase			):
				# FIXME: add fast tests
				pass







@nightly
@require_onnxruntime
@require_torch_gpu
class 			__A( unittest.TestCase			):




				@property
				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      List[str]:


								'''simple docstring'''




								return (
								    "CUDAExecutionProvider",
								    {
								        "gpu_mem_limit": "15000000000",  # 15GB
								        "arena_extend_strategy": "kSameAsRequested",
								    },
								)




				@property
				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      Union[str, Any]:


								'''simple docstring'''




								__a										=    ort.SessionOptions()
								__a										=    False
								return options




				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      Optional[Any]:


								'''simple docstring'''




								__a										=    load_image(
								    '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'''
								    '''/in_paint/overture-creations-5sI6fQgYIuo.png'''				)
								__a										=    load_image(
								    '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'''
								    '''/in_paint/overture-creations-5sI6fQgYIuo_mask.png'''				)
								__a										=    OnnxStableDiffusionInpaintPipeline.from_pretrained(
								    '''runwayml/stable-diffusion-inpainting'''		,     revision='''onnx'''		,     safety_checker=_snake_case		,     feature_extractor=_snake_case		,     provider=self.gpu_provider		,     sess_options=self.gpu_options		,     )
								pipe.set_progress_bar_config(disable=_snake_case				)

								__a										=    '''A red cat sitting on a park bench'''

								__a										=    np.random.RandomState(0				)
								__a										=    pipe(
								    prompt=_snake_case		,     image=_snake_case		,     mask_image=_snake_case		,     guidance_scale=7.5		,     num_inference_steps=10		,     generator=_snake_case		,     output_type='''np'''		,     )
								__a										=    output.images
								__a										=    images[0, 255:258, 255:258, -1]

								assert images.shape == (1, 512, 512, 3)
								__a										=    np.array([0.2514, 0.3007, 0.3517, 0.1790, 0.2382, 0.3167, 0.1944, 0.2273, 0.2464]				)

								assert np.abs(image_slice.flatten() - expected_slice				).max() < 1E-3




				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      Tuple:


								'''simple docstring'''




								__a										=    load_image(
								    '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'''
								    '''/in_paint/overture-creations-5sI6fQgYIuo.png'''				)
								__a										=    load_image(
								    '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'''
								    '''/in_paint/overture-creations-5sI6fQgYIuo_mask.png'''				)
								__a										=    LMSDiscreteScheduler.from_pretrained(
								    '''runwayml/stable-diffusion-inpainting'''		,     subfolder='''scheduler'''		,     revision='''onnx'''				)
								__a										=    OnnxStableDiffusionInpaintPipeline.from_pretrained(
								    '''runwayml/stable-diffusion-inpainting'''		,     revision='''onnx'''		,     scheduler=_snake_case		,     safety_checker=_snake_case		,     feature_extractor=_snake_case		,     provider=self.gpu_provider		,     sess_options=self.gpu_options		,     )
								pipe.set_progress_bar_config(disable=_snake_case				)

								__a										=    '''A red cat sitting on a park bench'''

								__a										=    np.random.RandomState(0				)
								__a										=    pipe(
								    prompt=_snake_case		,     image=_snake_case		,     mask_image=_snake_case		,     guidance_scale=7.5		,     num_inference_steps=20		,     generator=_snake_case		,     output_type='''np'''		,     )
								__a										=    output.images
								__a										=    images[0, 255:258, 255:258, -1]

								assert images.shape == (1, 512, 512, 3)
								__a										=    np.array([0.0086, 0.0077, 0.0083, 0.0093, 0.0107, 0.0139, 0.0094, 0.0097, 0.0125]				)

								assert np.abs(image_slice.flatten() - expected_slice				).max() < 1E-3