import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


UpperCamelCase  							=						re.compile(R'''\b(a|an|the)\b''', re.UNICODE)

UpperCamelCase  							=						None

def 			lowercase_				(			):
    lowercase__ :   List[Any]    					=				argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0.")
    parser.add_argument("data_file"		,	metavar="data.json"		,	help="Input data JSON file.")
    parser.add_argument("pred_file"		,	metavar="pred.json"		,	help="Model predictions.")
    parser.add_argument(
        "--out-file"		,	"-o"		,	metavar="eval.json"		,	help="Write accuracy metrics to file (default is stdout).")
    parser.add_argument(
        "--na-prob-file"		,	"-n"		,	metavar="na_prob.json"		,	help="Model estimates of probability of no answer.")
    parser.add_argument(
        "--na-prob-thresh"		,	"-t"		,	type=_lowerCamelCase		,	default=1.0		,	help="Predict \"\" if no-answer probability exceeds this (default = 1.0)."		,	)
    parser.add_argument(
        "--out-image-dir"		,	"-p"		,	metavar="out_images"		,	default=_lowerCamelCase		,	help="Save precision-recall curves to directory.")
    parser.add_argument("--verbose"		,	"-v"		,	action="store_true")
    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit(1)
    return parser.parse_args()

def 			lowercase_				(			_lowerCamelCase							:       List[Any]):
    lowercase__ :   Union[str, Any]    					=				{}
    for article in dataset:
        for p in article["paragraphs"]:
            for qa in p["qas"]:
                lowercase__ :   Optional[int]    					=				bool(qa["answers"]["text"])
    return qid_to_has_ans

def 			lowercase_				(			_lowerCamelCase							:       Union[str, Any]):

    def remove_articles(_lowerCamelCase							:       List[str]):
        return ARTICLES_REGEX.sub(" "		,	_lowerCamelCase)

    def white_space_fix(_lowerCamelCase							:       int):
        return " ".join(text.split())

    def remove_punc(_lowerCamelCase							:       Tuple):
        lowercase__ :   Any    					=				set(string.punctuation)
        return "".join(ch for ch in text if ch not in exclude)

    def lower(_lowerCamelCase							:       Any):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(_lowerCamelCase))))

def 			lowercase_				(			_lowerCamelCase							:       str):
    if not s:
        return []
    return normalize_answer(_lowerCamelCase).split()

def 			lowercase_				(			_lowerCamelCase							:       Optional[int]		,	_lowerCamelCase							:       Any):
    return int(normalize_answer(_lowerCamelCase) == normalize_answer(_lowerCamelCase))

def 			lowercase_				(			_lowerCamelCase							:       Union[str, Any]		,	_lowerCamelCase							:       Optional[Any]):
    lowercase__ :   str    					=				get_tokens(_lowerCamelCase)
    lowercase__ :   int    					=				get_tokens(_lowerCamelCase)
    lowercase__ :   List[Any]    					=				collections.Counter(_lowerCamelCase) & collections.Counter(_lowerCamelCase)
    lowercase__ :   List[Any]    					=				sum(common.values())
    if len(_lowerCamelCase) == 0 or len(_lowerCamelCase) == 0:
        # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
        return int(gold_toks == pred_toks)
    if num_same == 0:
        return 0
    lowercase__ :   Union[str, Any]    					=				1.0 * num_same / len(_lowerCamelCase)
    lowercase__ :   Optional[Any]    					=				1.0 * num_same / len(_lowerCamelCase)
    lowercase__ :   Dict    					=				(2 * precision * recall) / (precision + recall)
    return fa

def 			lowercase_				(			_lowerCamelCase							:       Tuple		,	_lowerCamelCase							:       Any):
    lowercase__ :   Union[str, Any]    					=				{}
    lowercase__ :   List[str]    					=				{}
    for article in dataset:
        for p in article["paragraphs"]:
            for qa in p["qas"]:
                lowercase__ :   List[Any]    					=				qa["id"]
                lowercase__ :   List[Any]    					=				[t for t in qa["answers"]["text"] if normalize_answer(_lowerCamelCase)]
                if not gold_answers:
                    # For unanswerable questions, only correct answer is empty string
                    lowercase__ :   List[str]    					=				[""]
                if qid not in preds:
                    print(f'''Missing prediction for {qid}''')
                    continue
                lowercase__ :   Any    					=				preds[qid]
                # Take max over all gold answers
                lowercase__ :   Tuple    					=				max(compute_exact(_lowerCamelCase		,	_lowerCamelCase) for a in gold_answers)
                lowercase__ :   Any    					=				max(compute_fa(_lowerCamelCase		,	_lowerCamelCase) for a in gold_answers)
    return exact_scores, fa_scores

def 			lowercase_				(			_lowerCamelCase							:       Union[str, Any]		,	_lowerCamelCase							:       Dict		,	_lowerCamelCase							:       Any		,	_lowerCamelCase							:       Any):
    lowercase__ :   Union[str, Any]    					=				{}
    for qid, s in scores.items():
        lowercase__ :   Optional[int]    					=				na_probs[qid] > na_prob_thresh
        if pred_na:
            lowercase__ :   Dict    					=				float(not qid_to_has_ans[qid])
        else:
            lowercase__ :   List[Any]    					=				s
    return new_scores

def 			lowercase_				(			_lowerCamelCase							:       Dict		,	_lowerCamelCase							:       Union[str, Any]		,	_lowerCamelCase							:       Union[str, Any]=None):
    if not qid_list:
        lowercase__ :   List[str]    					=				len(_lowerCamelCase)
        return collections.OrderedDict(
            [
                ("exact", 100.0 * sum(exact_scores.values()) / total),
                ("f1", 100.0 * sum(fa_scores.values()) / total),
                ("total", total),
            ])
    else:
        lowercase__ :   Union[str, Any]    					=				len(_lowerCamelCase)
        return collections.OrderedDict(
            [
                ("exact", 100.0 * sum(exact_scores[k] for k in qid_list) / total),
                ("f1", 100.0 * sum(fa_scores[k] for k in qid_list) / total),
                ("total", total),
            ])

def 			lowercase_				(			_lowerCamelCase							:       Dict		,	_lowerCamelCase							:       Any		,	_lowerCamelCase							:       Any):
    for k in new_eval:
        lowercase__ :   Union[str, Any]    					=				new_eval[k]

def 			lowercase_				(			_lowerCamelCase							:       Optional[Any]		,	_lowerCamelCase							:       Optional[int]		,	_lowerCamelCase							:       Tuple		,	_lowerCamelCase							:       List[Any]):
    plt.step(_lowerCamelCase		,	_lowerCamelCase		,	color="b"		,	alpha=0.2		,	where="post")
    plt.fill_between(_lowerCamelCase		,	_lowerCamelCase		,	step="post"		,	alpha=0.2		,	color="b")
    plt.xlabel("Recall")
    plt.ylabel("Precision")
    plt.xlim([0.0, 1.05])
    plt.ylim([0.0, 1.05])
    plt.title(_lowerCamelCase)
    plt.savefig(_lowerCamelCase)
    plt.clf()

def 			lowercase_				(			_lowerCamelCase							:       Any		,	_lowerCamelCase							:       List[Any]		,	_lowerCamelCase							:       int		,	_lowerCamelCase							:       List[Any]		,	_lowerCamelCase							:       Dict=None		,	_lowerCamelCase							:       Optional[int]=None):
    lowercase__ :   Optional[Any]    					=				sorted(_lowerCamelCase		,	key=lambda _lowerCamelCase: na_probs[k])
    lowercase__ :   Optional[int]    					=				0.0
    lowercase__ :   str    					=				1.0
    lowercase__ :   List[str]    					=				0.0
    lowercase__ :   List[str]    					=				[1.0]
    lowercase__ :   Union[str, Any]    					=				[0.0]
    lowercase__ :   Dict    					=				0.0
    for i, qid in enumerate(_lowerCamelCase):
        if qid_to_has_ans[qid]:
            true_pos += scores[qid]
        lowercase__ :   str    					=				true_pos / float(i + 1)
        lowercase__ :   List[str]    					=				true_pos / float(_lowerCamelCase)
        if i == len(_lowerCamelCase) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
            # i.e., if we can put a threshold after this point
            avg_prec += cur_p * (cur_r - recalls[-1])
            precisions.append(_lowerCamelCase)
            recalls.append(_lowerCamelCase)
    if out_image:
        plot_pr_curve(_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase)
    return {"ap": 100.0 * avg_prec}

def 			lowercase_				(			_lowerCamelCase							:       str		,	_lowerCamelCase							:       Optional[Any]		,	_lowerCamelCase							:       int		,	_lowerCamelCase							:       Tuple		,	_lowerCamelCase							:       Union[str, Any]		,	_lowerCamelCase							:       Optional[Any]):
    if out_image_dir and not os.path.exists(_lowerCamelCase):
        os.makedirs(_lowerCamelCase)
    lowercase__ :   Union[str, Any]    					=				sum(1 for v in qid_to_has_ans.values() if v)
    if num_true_pos == 0:
        return
    lowercase__ :   Tuple    					=				make_precision_recall_eval(
        _lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	out_image=os.path.join(_lowerCamelCase		,	"pr_exact.png")		,	title="Precision-Recall curve for Exact Match score"		,	)
    lowercase__ :   Union[str, Any]    					=				make_precision_recall_eval(
        _lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	out_image=os.path.join(_lowerCamelCase		,	"pr_f1.png")		,	title="Precision-Recall curve for F1 score"		,	)
    lowercase__ :   Union[str, Any]    					=				{k: float(_lowerCamelCase) for k, v in qid_to_has_ans.items()}
    lowercase__ :   Optional[int]    					=				make_precision_recall_eval(
        _lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	out_image=os.path.join(_lowerCamelCase		,	"pr_oracle.png")		,	title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"		,	)
    merge_eval(_lowerCamelCase		,	_lowerCamelCase		,	"pr_exact")
    merge_eval(_lowerCamelCase		,	_lowerCamelCase		,	"pr_f1")
    merge_eval(_lowerCamelCase		,	_lowerCamelCase		,	"pr_oracle")

def 			lowercase_				(			_lowerCamelCase							:       Tuple		,	_lowerCamelCase							:       List[Any]		,	_lowerCamelCase							:       Optional[Any]		,	_lowerCamelCase							:       List[Any]):
    if not qid_list:
        return
    lowercase__ :   List[str]    					=				[na_probs[k] for k in qid_list]
    lowercase__ :   Tuple    					=				np.ones_like(_lowerCamelCase) / float(len(_lowerCamelCase))
    plt.hist(_lowerCamelCase		,	weights=_lowerCamelCase		,	bins=20		,	range=(0.0, 1.0))
    plt.xlabel("Model probability of no-answer")
    plt.ylabel("Proportion of dataset")
    plt.title(f'''Histogram of no-answer probability: {name}''')
    plt.savefig(os.path.join(_lowerCamelCase		,	f'''na_prob_hist_{name}.png'''))
    plt.clf()

def 			lowercase_				(			_lowerCamelCase							:       List[str]		,	_lowerCamelCase							:       Any		,	_lowerCamelCase							:       Any		,	_lowerCamelCase							:       Optional[int]):
    lowercase__ :   Dict    					=				sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k])
    lowercase__ :   Union[str, Any]    					=				num_no_ans
    lowercase__ :   Optional[int]    					=				cur_score
    lowercase__ :   List[str]    					=				0.0
    lowercase__ :   List[Any]    					=				sorted(_lowerCamelCase		,	key=lambda _lowerCamelCase: na_probs[k])
    for i, qid in enumerate(_lowerCamelCase):
        if qid not in scores:
            continue
        if qid_to_has_ans[qid]:
            lowercase__ :   List[Any]    					=				scores[qid]
        else:
            if preds[qid]:
                lowercase__ :   str    					=				-1
            else:
                lowercase__ :   List[Any]    					=				0
        cur_score += diff
        if cur_score > best_score:
            lowercase__ :   Any    					=				cur_score
            lowercase__ :   Dict    					=				na_probs[qid]
    return 100.0 * best_score / len(_lowerCamelCase), best_thresh

def 			lowercase_				(			_lowerCamelCase							:       Optional[int]		,	_lowerCamelCase							:       str		,	_lowerCamelCase							:       List[str]		,	_lowerCamelCase							:       int		,	_lowerCamelCase							:       Optional[Any]		,	_lowerCamelCase							:       Optional[Any]):
    lowercase__					,   lowercase__ :   int    					=				find_best_thresh(_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase)
    lowercase__					,   lowercase__ :   List[str]    					=				find_best_thresh(_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase)
    lowercase__ :   Optional[Any]    					=				best_exact
    lowercase__ :   int    					=				exact_thresh
    lowercase__ :   Tuple    					=				best_fa
    lowercase__ :   Optional[Any]    					=				fa_thresh

def 			lowercase_				(			):
    with open(OPTS.data_file) as f:
        lowercase__ :   List[Any]    					=				json.load(_lowerCamelCase)
        lowercase__ :   Union[str, Any]    					=				dataset_json["data"]
    with open(OPTS.pred_file) as f:
        lowercase__ :   Optional[int]    					=				json.load(_lowerCamelCase)
    if OPTS.na_prob_file:
        with open(OPTS.na_prob_file) as f:
            lowercase__ :   Union[str, Any]    					=				json.load(_lowerCamelCase)
    else:
        lowercase__ :   List[str]    					=				{k: 0.0 for k in preds}
    lowercase__ :   Tuple    					=				make_qid_to_has_ans(_lowerCamelCase)  # maps qid to True/False
    lowercase__ :   int    					=				[k for k, v in qid_to_has_ans.items() if v]
    lowercase__ :   str    					=				[k for k, v in qid_to_has_ans.items() if not v]
    lowercase__					,   lowercase__ :   Any    					=				get_raw_scores(_lowerCamelCase		,	_lowerCamelCase)
    lowercase__ :   int    					=				apply_no_ans_threshold(_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	OPTS.na_prob_thresh)
    lowercase__ :   int    					=				apply_no_ans_threshold(_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	OPTS.na_prob_thresh)
    lowercase__ :   Dict    					=				make_eval_dict(_lowerCamelCase		,	_lowerCamelCase)
    if has_ans_qids:
        lowercase__ :   Dict    					=				make_eval_dict(_lowerCamelCase		,	_lowerCamelCase		,	qid_list=_lowerCamelCase)
        merge_eval(_lowerCamelCase		,	_lowerCamelCase		,	"HasAns")
    if no_ans_qids:
        lowercase__ :   str    					=				make_eval_dict(_lowerCamelCase		,	_lowerCamelCase		,	qid_list=_lowerCamelCase)
        merge_eval(_lowerCamelCase		,	_lowerCamelCase		,	"NoAns")
    if OPTS.na_prob_file:
        find_all_best_thresh(_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase)
    if OPTS.na_prob_file and OPTS.out_image_dir:
        run_precision_recall_analysis(_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	OPTS.out_image_dir)
        histogram_na_prob(_lowerCamelCase		,	_lowerCamelCase		,	OPTS.out_image_dir		,	"hasAns")
        histogram_na_prob(_lowerCamelCase		,	_lowerCamelCase		,	OPTS.out_image_dir		,	"noAns")
    if OPTS.out_file:
        with open(OPTS.out_file		,	"w") as f:
            json.dump(_lowerCamelCase		,	_lowerCamelCase)
    else:
        print(json.dumps(_lowerCamelCase		,	indent=2))


if __name__ == "__main__":
      UpperCamelCase  							=						parse_args()
      if OPTS.out_image_dir:
            import matplotlib

            matplotlib.use('''Agg''')
            import matplotlib.pyplot as plt
      main()








'''simple docstring'''






# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import sys

import transformers


UpperCamelCase__:   Optional[int]								=			"3"

print("Python version:", sys.version)
print("transformers version:", transformers.__version__)

try:
						import torch

						print("Torch version:", torch.__version__)
						print("Cuda available:", torch.cuda.is_available())
						print("Cuda version:", torch.version.cuda)
						print("CuDNN version:", torch.backends.cudnn.version())
						print("Number of GPUs available:", torch.cuda.device_count())
						print("NCCL version:", torch.cuda.nccl.version())
except ImportError:
						print("Torch version:", None)

try:
						import deepspeed

						print("DeepSpeed version:", deepspeed.__version__)
except ImportError:
						print("DeepSpeed version:", None)

try:
						import tensorflow as tf

						print("TensorFlow version:", tf.__version__)
						print("TF GPUs available:", bool(tf.config.list_physical_devices("GPU")))
						print("Number of TF GPUs available:", len(tf.config.list_physical_devices("GPU")))
except ImportError:
						print("TensorFlow version:", None)





