
"""simple docstring"""







import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowerCamelCase__   					=		re.compile(R"""\b(a|an|the)\b""", re.UNICODE)

lowerCamelCase__   					=		None
def     __lowerCAmelCase				():
	__lowerCAmelCase			:						Dict									=      argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.'		)
	parser.add_argument('data_file'       ,  metavar='data.json'       ,  help='Input data JSON file.'		)
	parser.add_argument('pred_file'       ,  metavar='pred.json'       ,  help='Model predictions.'		)
	parser.add_argument(
	    '--out-file'       ,  '-o'       ,  metavar='eval.json'       ,  help='Write accuracy metrics to file (default is stdout).'		)
	parser.add_argument(
	    '--na-prob-file'       ,  '-n'       ,  metavar='na_prob.json'       ,  help='Model estimates of probability of no answer.'		)
	parser.add_argument(
	    '--na-prob-thresh'       ,  '-t'       ,  type=_a       ,  default=1.0       ,  help='Predict "" if no-answer probability exceeds this (default = 1.0).'       ,  )
	parser.add_argument(
	    '--out-image-dir'       ,  '-p'       ,  metavar='out_images'       ,  default=_a       ,  help='Save precision-recall curves to directory.'		)
	parser.add_argument('--verbose'       ,  '-v'       ,  action='store_true'		)
	if len(sys.argv		) == 1:
		parser.print_help()
		sys.exit(1		)
	return parser.parse_args()
def     __lowerCAmelCase				(_UpperCamelCase		):
	__lowerCAmelCase			:						Union[str, Any]									=      {}
	for article in dataset:
		for p in article["paragraphs"]:
			for qa in p["qas"]:
				__lowerCAmelCase			:						Any									=      bool(qa['answers']['text']		)
	return qid_to_has_ans
def     __lowerCAmelCase				(_UpperCamelCase		):

	def remove_articles(_UpperCamelCase		):
		return ARTICLES_REGEX.sub(' '       ,  _a		)

	def white_space_fix(_UpperCamelCase		):
		return " ".join(text.split()		)

	def remove_punc(_UpperCamelCase		):
		__lowerCAmelCase			:						List[Any]									=      set(string.punctuation		)
		return "".join(ch for ch in text if ch not in exclude		)

	def lower(_UpperCamelCase		):
		return text.lower()

	return white_space_fix(remove_articles(remove_punc(lower(_a		)		)		)		)
def     __lowerCAmelCase				(_UpperCamelCase		):
	if not s:
		return []
	return normalize_answer(_a		).split()
def     __lowerCAmelCase				(_UpperCamelCase       ,  _UpperCamelCase		):
	return int(normalize_answer(_a		) == normalize_answer(_a		)		)
def     __lowerCAmelCase				(_UpperCamelCase       ,  _UpperCamelCase		):
	__lowerCAmelCase			:						List[Any]									=      get_tokens(_a		)
	__lowerCAmelCase			:						Any									=      get_tokens(_a		)
	__lowerCAmelCase			:						Union[str, Any]									=      collections.Counter(_a		) & collections.Counter(_a		)
	__lowerCAmelCase			:						Union[str, Any]									=      sum(common.values()		)
	if len(_a		) == 0 or len(_a		) == 0:
		# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
		return int(gold_toks == pred_toks		)
	if num_same == 0:
		return 0
	__lowerCAmelCase			:						Optional[int]									=      1.0 * num_same / len(_a		)
	__lowerCAmelCase			:						str									=      1.0 * num_same / len(_a		)
	__lowerCAmelCase			:						str									=      (2 * precision * recall) / (precision + recall)
	return fa
def     __lowerCAmelCase				(_UpperCamelCase       ,  _UpperCamelCase		):
	__lowerCAmelCase			:						int									=      {}
	__lowerCAmelCase			:						Optional[int]									=      {}
	for article in dataset:
		for p in article["paragraphs"]:
			for qa in p["qas"]:
				__lowerCAmelCase			:						str									=      qa['id']
				__lowerCAmelCase			:						str									=      [t for t in qa['answers']['text'] if normalize_answer(_a		)]
				if not gold_answers:
					# For unanswerable questions, only correct answer is empty string
					__lowerCAmelCase			:						Optional[int]									=      ['']
				if qid not in preds:
					print(F"Missing prediction for {qid}"		)
					continue
				__lowerCAmelCase			:						int									=      preds[qid]
				# Take max over all gold answers
				__lowerCAmelCase			:						Union[str, Any]									=      max(compute_exact(_a       ,  _a		) for a in gold_answers		)
				__lowerCAmelCase			:						Optional[Any]									=      max(compute_fa(_a       ,  _a		) for a in gold_answers		)
	return exact_scores, fa_scores
def     __lowerCAmelCase				(_UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase		):
	__lowerCAmelCase			:						Dict									=      {}
	for qid, s in scores.items():
		__lowerCAmelCase			:						Dict									=      na_probs[qid] > na_prob_thresh
		if pred_na:
			__lowerCAmelCase			:						List[str]									=      float(not qid_to_has_ans[qid]		)
		else:
			__lowerCAmelCase			:						int									=      s
	return new_scores
def     __lowerCAmelCase				(_UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase=None		):
	if not qid_list:
		__lowerCAmelCase			:						Union[str, Any]									=      len(_a		)
		return collections.OrderedDict(
		    [
		        ('exact', 100.0 * sum(exact_scores.values()		) / total),
		        ('f1', 100.0 * sum(fa_scores.values()		) / total),
		        ('total', total),
		    ]		)
	else:
		__lowerCAmelCase			:						str									=      len(_a		)
		return collections.OrderedDict(
		    [
		        ('exact', 100.0 * sum(exact_scores[k] for k in qid_list		) / total),
		        ('f1', 100.0 * sum(fa_scores[k] for k in qid_list		) / total),
		        ('total', total),
		    ]		)
def     __lowerCAmelCase				(_UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase		):
	for k in new_eval:
		__lowerCAmelCase			:						str									=      new_eval[k]
def     __lowerCAmelCase				(_UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase		):
	plt.step(_a       ,  _a       ,  color='b'       ,  alpha=0.2       ,  where='post'		)
	plt.fill_between(_a       ,  _a       ,  step='post'       ,  alpha=0.2       ,  color='b'		)
	plt.xlabel('Recall'		)
	plt.ylabel('Precision'		)
	plt.xlim([0.0, 1.05]		)
	plt.ylim([0.0, 1.05]		)
	plt.title(_a		)
	plt.savefig(_a		)
	plt.clf()
def     __lowerCAmelCase				(_UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase=None       ,  _UpperCamelCase=None		):
	__lowerCAmelCase			:						str									=      sorted(_a       ,  key=lambda _UpperCamelCase		: na_probs[k]		)
	__lowerCAmelCase			:						Optional[Any]									=      0.0
	__lowerCAmelCase			:						Tuple									=      1.0
	__lowerCAmelCase			:						Any									=      0.0
	__lowerCAmelCase			:						Union[str, Any]									=      [1.0]
	__lowerCAmelCase			:						Any									=      [0.0]
	__lowerCAmelCase			:						Tuple									=      0.0
	for i, qid in enumerate(_a		):
		if qid_to_has_ans[qid]:
			true_pos += scores[qid]
		__lowerCAmelCase			:						Optional[Any]									=      true_pos / float(i + 1		)
		__lowerCAmelCase			:						Optional[int]									=      true_pos / float(_a		)
		if i == len(_a		) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
			# i.e., if we can put a threshold after this point
			avg_prec += cur_p * (cur_r - recalls[-1])
			precisions.append(_a		)
			recalls.append(_a		)
	if out_image:
		plot_pr_curve(_a       ,  _a       ,  _a       ,  _a		)
	return {"ap": 100.0 * avg_prec}
def     __lowerCAmelCase				(_UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase		):
	if out_image_dir and not os.path.exists(_a		):
		os.makedirs(_a		)
	__lowerCAmelCase			:						Optional[Any]									=      sum(1 for v in qid_to_has_ans.values() if v		)
	if num_true_pos == 0:
		return
	__lowerCAmelCase			:						int									=      make_precision_recall_eval(
	    _a       ,  _a       ,  _a       ,  _a       ,  out_image=os.path.join(_a       ,  'pr_exact.png'		)       ,  title='Precision-Recall curve for Exact Match score'       ,  )
	__lowerCAmelCase			:						List[Any]									=      make_precision_recall_eval(
	    _a       ,  _a       ,  _a       ,  _a       ,  out_image=os.path.join(_a       ,  'pr_f1.png'		)       ,  title='Precision-Recall curve for F1 score'       ,  )
	__lowerCAmelCase			:						Union[str, Any]									=      {k: float(_a		) for k, v in qid_to_has_ans.items()}
	__lowerCAmelCase			:						Optional[int]									=      make_precision_recall_eval(
	    _a       ,  _a       ,  _a       ,  _a       ,  out_image=os.path.join(_a       ,  'pr_oracle.png'		)       ,  title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'       ,  )
	merge_eval(_a       ,  _a       ,  'pr_exact'		)
	merge_eval(_a       ,  _a       ,  'pr_f1'		)
	merge_eval(_a       ,  _a       ,  'pr_oracle'		)
def     __lowerCAmelCase				(_UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase		):
	if not qid_list:
		return
	__lowerCAmelCase			:						Tuple									=      [na_probs[k] for k in qid_list]
	__lowerCAmelCase			:						Optional[int]									=      np.ones_like(_a		) / float(len(_a		)		)
	plt.hist(_a       ,  weights=_a       ,  bins=20       ,  range=(0.0, 1.0)		)
	plt.xlabel('Model probability of no-answer'		)
	plt.ylabel('Proportion of dataset'		)
	plt.title(F"Histogram of no-answer probability: {name}"		)
	plt.savefig(os.path.join(_a       ,  F"na_prob_hist_{name}.png"		)		)
	plt.clf()
def     __lowerCAmelCase				(_UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase		):
	__lowerCAmelCase			:						List[Any]									=      sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]		)
	__lowerCAmelCase			:						List[Any]									=      num_no_ans
	__lowerCAmelCase			:						Any									=      cur_score
	__lowerCAmelCase			:						Any									=      0.0
	__lowerCAmelCase			:						Dict									=      sorted(_a       ,  key=lambda _UpperCamelCase		: na_probs[k]		)
	for i, qid in enumerate(_a		):
		if qid not in scores:
			continue
		if qid_to_has_ans[qid]:
			__lowerCAmelCase			:						Union[str, Any]									=      scores[qid]
		else:
			if preds[qid]:
				__lowerCAmelCase			:						int									=      -1
			else:
				__lowerCAmelCase			:						Optional[int]									=      0
		cur_score += diff
		if cur_score > best_score:
			__lowerCAmelCase			:						List[Any]									=      cur_score
			__lowerCAmelCase			:						Any									=      na_probs[qid]
	return 100.0 * best_score / len(_a		), best_thresh
def     __lowerCAmelCase				(_UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase		):
	__lowerCAmelCase			,   __lowerCAmelCase			:						List[Any]									=      find_best_thresh(_a       ,  _a       ,  _a       ,  _a		)
	__lowerCAmelCase			,   __lowerCAmelCase			:						Optional[int]									=      find_best_thresh(_a       ,  _a       ,  _a       ,  _a		)
	__lowerCAmelCase			:						List[Any]									=      best_exact
	__lowerCAmelCase			:						str									=      exact_thresh
	__lowerCAmelCase			:						Any									=      best_fa
	__lowerCAmelCase			:						Union[str, Any]									=      fa_thresh
def     __lowerCAmelCase				():
	with open(OPTS.data_file		) as f:
		__lowerCAmelCase			:						Any									=      json.load(_a		)
		__lowerCAmelCase			:						str									=      dataset_json['data']
	with open(OPTS.pred_file		) as f:
		__lowerCAmelCase			:						Union[str, Any]									=      json.load(_a		)
	if OPTS.na_prob_file:
		with open(OPTS.na_prob_file		) as f:
			__lowerCAmelCase			:						Union[str, Any]									=      json.load(_a		)
	else:
		__lowerCAmelCase			:						Tuple									=      {k: 0.0 for k in preds}
	__lowerCAmelCase			:						Optional[Any]									=      make_qid_to_has_ans(_a		)  # maps qid to True/False
	__lowerCAmelCase			:						Any									=      [k for k, v in qid_to_has_ans.items() if v]
	__lowerCAmelCase			:						int									=      [k for k, v in qid_to_has_ans.items() if not v]
	__lowerCAmelCase			,   __lowerCAmelCase			:						Optional[Any]									=      get_raw_scores(_a       ,  _a		)
	__lowerCAmelCase			:						Union[str, Any]									=      apply_no_ans_threshold(_a       ,  _a       ,  _a       ,  OPTS.na_prob_thresh		)
	__lowerCAmelCase			:						str									=      apply_no_ans_threshold(_a       ,  _a       ,  _a       ,  OPTS.na_prob_thresh		)
	__lowerCAmelCase			:						Tuple									=      make_eval_dict(_a       ,  _a		)
	if has_ans_qids:
		__lowerCAmelCase			:						Dict									=      make_eval_dict(_a       ,  _a       ,  qid_list=_a		)
		merge_eval(_a       ,  _a       ,  'HasAns'		)
	if no_ans_qids:
		__lowerCAmelCase			:						List[str]									=      make_eval_dict(_a       ,  _a       ,  qid_list=_a		)
		merge_eval(_a       ,  _a       ,  'NoAns'		)
	if OPTS.na_prob_file:
		find_all_best_thresh(_a       ,  _a       ,  _a       ,  _a       ,  _a       ,  _a		)
	if OPTS.na_prob_file and OPTS.out_image_dir:
		run_precision_recall_analysis(_a       ,  _a       ,  _a       ,  _a       ,  _a       ,  OPTS.out_image_dir		)
		histogram_na_prob(_a       ,  _a       ,  OPTS.out_image_dir       ,  'hasAns'		)
		histogram_na_prob(_a       ,  _a       ,  OPTS.out_image_dir       ,  'noAns'		)
	if OPTS.out_file:
		with open(OPTS.out_file       ,  'w'		) as f:
			json.dump(_a       ,  _a		)
	else:
		print(json.dumps(_a       ,  indent=2		)		)


if __name__ == "__main__":
							lowerCamelCase__   					=		parse_args()
							if OPTS.out_image_dir:
														import matplotlib

														matplotlib.use("""Agg""")
														import matplotlib.pyplot as plt
							main()





'''simple docstring'''







import argparse
import json
import logging
import os
import sys
from unittest.mock import patch

from transformers.testing_utils import TestCasePlus, get_gpu_count, slow


A     =[
    os.path.join(os.path.dirname(__file__), dirname)
    for dirname in [
        'text-classification',
        'language-modeling',
        'summarization',
        'token-classification',
        'question-answering',
    ]
]
sys.path.extend(SRC_DIRS)


if SRC_DIRS is not None:
   import run_clm_flax
   import run_flax_glue
   import run_flax_ner
   import run_mlm_flax
   import run_qa
   import run_summarization_flax
   import run_ta_mlm_flax


logging.basicConfig(level=logging.DEBUG)

A     =logging.getLogger()





def 						snake_case_							():
    UpperCAmelCase			       =		argparse.ArgumentParser()
    parser.add_argument('''-f'''     )
    UpperCAmelCase			       =		parser.parse_args()
    return args.f





def 						snake_case_							(_a	:							List[str]    ,   _a	:							Union[str, Any]="eval"     ):
    UpperCAmelCase			       =		os.path.join(_a    ,   F"{split}_results.json"     )
    if os.path.exists(_a     ):
        with open(_a    ,   '''r'''     ) as f:
            return json.load(_a     )
    raise ValueError(F"can't find {path}"     )


A     =logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)







class 			_a  (	__a ):
       def     A      (	self						:	Any				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.get_auto_remove_tmp_dir()
           UpperCAmelCase			       =		f"\n            run_glue.py\n            --model_name_or_path distilbert-base-uncased\n            --output_dir {tmp_dir}\n            --train_file ./tests/fixtures/tests_samples/MRPC/train.csv\n            --validation_file ./tests/fixtures/tests_samples/MRPC/dev.csv\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --learning_rate=1e-4\n            --eval_steps=2\n            --warmup_steps=2\n            --seed=42\n            --max_seq_length=128\n            ".split()

           with patch.object(lowercase   ,				'''argv'''   ,				lowercase				):
               run_flax_glue.main()
               UpperCAmelCase			       =		get_results(lowercase				)
               self.assertGreaterEqual(result['''eval_accuracy''']   ,				0.75				)
       @slow
       def     A      (	self						:	Any				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.get_auto_remove_tmp_dir()
           UpperCAmelCase			       =		f"\n            run_clm_flax.py\n            --model_name_or_path distilgpt2\n            --train_file ./tests/fixtures/sample_text.txt\n            --validation_file ./tests/fixtures/sample_text.txt\n            --do_train\n            --do_eval\n            --block_size 128\n            --per_device_train_batch_size 4\n            --per_device_eval_batch_size 4\n            --num_train_epochs 2\n            --logging_steps 2 --eval_steps 2\n            --output_dir {tmp_dir}\n            --overwrite_output_dir\n            ".split()

           with patch.object(lowercase   ,				'''argv'''   ,				lowercase				):
               run_clm_flax.main()
               UpperCAmelCase			       =		get_results(lowercase				)
               self.assertLess(result['''eval_perplexity''']   ,				100				)
       @slow
       def     A      (	self						:	str				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.get_auto_remove_tmp_dir()
           UpperCAmelCase			       =		f"\n            run_summarization.py\n            --model_name_or_path t5-small\n            --train_file tests/fixtures/tests_samples/xsum/sample.json\n            --validation_file tests/fixtures/tests_samples/xsum/sample.json\n            --test_file tests/fixtures/tests_samples/xsum/sample.json\n            --output_dir {tmp_dir}\n            --overwrite_output_dir\n            --num_train_epochs=3\n            --warmup_steps=8\n            --do_train\n            --do_eval\n            --do_predict\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --predict_with_generate\n        ".split()

           with patch.object(lowercase   ,				'''argv'''   ,				lowercase				):
               run_summarization_flax.main()
               UpperCAmelCase			       =		get_results(lowercase   ,				split='''test'''				)
               self.assertGreaterEqual(result['''test_rouge1''']   ,				10				)
               self.assertGreaterEqual(result['''test_rouge2''']   ,				2				)
               self.assertGreaterEqual(result['''test_rougeL''']   ,				7				)
               self.assertGreaterEqual(result['''test_rougeLsum''']   ,				7				)
       @slow
       def     A      (	self						:	int				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.get_auto_remove_tmp_dir()
           UpperCAmelCase			       =		f"\n            run_mlm.py\n            --model_name_or_path distilroberta-base\n            --train_file ./tests/fixtures/sample_text.txt\n            --validation_file ./tests/fixtures/sample_text.txt\n            --output_dir {tmp_dir}\n            --overwrite_output_dir\n            --max_seq_length 128\n            --per_device_train_batch_size 4\n            --per_device_eval_batch_size 4\n            --logging_steps 2 --eval_steps 2\n            --do_train\n            --do_eval\n            --num_train_epochs=1\n        ".split()

           with patch.object(lowercase   ,				'''argv'''   ,				lowercase				):
               run_mlm_flax.main()
               UpperCAmelCase			       =		get_results(lowercase				)
               self.assertLess(result['''eval_perplexity''']   ,				42				)
       @slow
       def     A      (	self						:	Tuple				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.get_auto_remove_tmp_dir()
           UpperCAmelCase			       =		f"\n            run_t5_mlm_flax.py\n            --model_name_or_path t5-small\n            --train_file ./tests/fixtures/sample_text.txt\n            --validation_file ./tests/fixtures/sample_text.txt\n            --do_train\n            --do_eval\n            --max_seq_length 128\n            --per_device_train_batch_size 4\n            --per_device_eval_batch_size 4\n            --num_train_epochs 2\n            --logging_steps 2 --eval_steps 2\n            --output_dir {tmp_dir}\n            --overwrite_output_dir\n            ".split()

           with patch.object(lowercase   ,				'''argv'''   ,				lowercase				):
               run_ta_mlm_flax.main()
               UpperCAmelCase			       =		get_results(lowercase				)
               self.assertGreaterEqual(result['''eval_accuracy''']   ,				0.42				)
       @slow
       def     A      (	self						:	Tuple				):



           '''simple docstring'''

           UpperCAmelCase			       =		7 if get_gpu_count() > 1 else 2

           UpperCAmelCase			       =		self.get_auto_remove_tmp_dir()
           UpperCAmelCase			       =		f"\n            run_flax_ner.py\n            --model_name_or_path bert-base-uncased\n            --train_file tests/fixtures/tests_samples/conll/sample.json\n            --validation_file tests/fixtures/tests_samples/conll/sample.json\n            --output_dir {tmp_dir}\n            --overwrite_output_dir\n            --do_train\n            --do_eval\n            --warmup_steps=2\n            --learning_rate=2e-4\n            --logging_steps 2 --eval_steps 2\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=2\n            --num_train_epochs={epochs}\n            --seed 7\n        ".split()

           with patch.object(lowercase   ,				'''argv'''   ,				lowercase				):
               run_flax_ner.main()
               UpperCAmelCase			       =		get_results(lowercase				)
               self.assertGreaterEqual(result['''eval_accuracy''']   ,				0.75				)
               self.assertGreaterEqual(result['''eval_f1''']   ,				0.3				)







       @slow
       def     A      (	self						:	Union[str, Any]				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.get_auto_remove_tmp_dir()
           UpperCAmelCase			       =		f"\n            run_qa.py\n            --model_name_or_path bert-base-uncased\n            --version_2_with_negative\n            --train_file tests/fixtures/tests_samples/SQUAD/sample.json\n            --validation_file tests/fixtures/tests_samples/SQUAD/sample.json\n            --output_dir {tmp_dir}\n            --overwrite_output_dir\n            --num_train_epochs=3\n            --warmup_steps=2\n            --do_train\n            --do_eval\n            --logging_steps 2 --eval_steps 2\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n        ".split()

           with patch.object(lowercase   ,				'''argv'''   ,				lowercase				):
               run_qa.main()
               UpperCAmelCase			       =		get_results(lowercase				)
               self.assertGreaterEqual(result['''eval_f1''']   ,				30				)
               self.assertGreaterEqual(result['''eval_exact''']   ,				30				)




