import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


_UpperCAmelCase						=		re.compile(r"""\b(a|an|the)\b""", re.UNICODE)

_UpperCAmelCase						=		None




def 				UpperCamelCase						(       ):
		'''simple docstring'''




		A_		:				Tuple     =      argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.'	)
		parser.add_argument('data_file'						,metavar='data.json'						,help='Input data JSON file.'	)
		parser.add_argument('pred_file'						,metavar='pred.json'						,help='Model predictions.'	)
		parser.add_argument(
		    '--out-file'						,'-o'						,metavar='eval.json'						,help='Write accuracy metrics to file (default is stdout).'	)
		parser.add_argument(
		    '--na-prob-file'						,'-n'						,metavar='na_prob.json'						,help='Model estimates of probability of no answer.'	)
		parser.add_argument(
		    '--na-prob-thresh'						,'-t'						,type=a_						,default=1.0						,help='Predict \"\" if no-answer probability exceeds this (default = 1.0).'						,)
		parser.add_argument(
		    '--out-image-dir'						,'-p'						,metavar='out_images'						,default=a_						,help='Save precision-recall curves to directory.'	)
		parser.add_argument('--verbose'						,'-v'						,action='store_true'	)
		if len(sys.argv	) == 1:
				parser.print_help()
				sys.exit(1	)
		return parser.parse_args()




def 				UpperCamelCase						(       __lowercase						: Optional[Any]	):
		'''simple docstring'''




		A_		:				int     =      {}
		for article in dataset:
				for p in article["paragraphs"]:
						for qa in p["qas"]:
								A_		:				Any     =      bool(qa['answers']['text']	)
		return qid_to_has_ans




def 				UpperCamelCase						(       __lowercase						: Dict	):
		'''simple docstring'''




		def remove_articles(__lowercase						: Optional[int]	):
				return ARTICLES_REGEX.sub(' '						,a_	)

		def white_space_fix(__lowercase						: Optional[Any]	):
				return " ".join(text.split()	)

		def remove_punc(__lowercase						: Tuple	):
				A_		:				Tuple     =      set(string.punctuation	)
				return "".join(ch for ch in text if ch not in exclude	)

		def lower(__lowercase						: Tuple	):
				return text.lower()

		return white_space_fix(remove_articles(remove_punc(lower(a_	)	)	)	)




def 				UpperCamelCase						(       __lowercase						: Dict	):
		'''simple docstring'''




		if not s:
				return []
		return normalize_answer(a_	).split()




def 				UpperCamelCase						(       __lowercase						: List[str]						,__lowercase						: Dict	):
		'''simple docstring'''




		return int(normalize_answer(a_	) == normalize_answer(a_	)	)




def 				UpperCamelCase						(       __lowercase						: Optional[Any]						,__lowercase						: Dict	):
		'''simple docstring'''




		A_		:				List[str]     =      get_tokens(a_	)
		A_		:				Optional[int]     =      get_tokens(a_	)
		A_		:				int     =      collections.Counter(a_	) & collections.Counter(a_	)
		A_		:				int     =      sum(common.values()	)
		if len(a_	) == 0 or len(a_	) == 0:
				# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
				return int(gold_toks == pred_toks	)
		if num_same == 0:
				return 0
		A_		:				Any     =      1.0 * num_same / len(a_	)
		A_		:				Optional[Any]     =      1.0 * num_same / len(a_	)
		A_		:				str     =      (2 * precision * recall) / (precision + recall)
		return fa




def 				UpperCamelCase						(       __lowercase						: Tuple						,__lowercase						: Tuple	):
		'''simple docstring'''




		A_		:				Union[str, Any]     =      {}
		A_		:				Tuple     =      {}
		for article in dataset:
				for p in article["paragraphs"]:
						for qa in p["qas"]:
								A_		:				int     =      qa['id']
								A_		:				Dict     =      [t for t in qa['answers']['text'] if normalize_answer(a_	)]
								if not gold_answers:
										# For unanswerable questions, only correct answer is empty string
										A_		:				List[Any]     =      ['']
								if qid not in preds:
										print(f'''Missing prediction for {qid}'''	)
										continue
								A_		:				Tuple     =      preds[qid]
								# Take max over all gold answers
								A_		:				Dict     =      max(compute_exact(a_						,a_	) for a in gold_answers	)
								A_		:				Dict     =      max(compute_fa(a_						,a_	) for a in gold_answers	)
		return exact_scores, fa_scores




def 				UpperCamelCase						(       __lowercase						: Optional[Any]						,__lowercase						: Optional[int]						,__lowercase						: List[Any]						,__lowercase						: Tuple	):
		'''simple docstring'''




		A_		:				str     =      {}
		for qid, s in scores.items():
				A_		:				Any     =      na_probs[qid] > na_prob_thresh
				if pred_na:
						A_		:				List[str]     =      float(not qid_to_has_ans[qid]	)
				else:
						A_		:				Optional[int]     =      s
		return new_scores




def 				UpperCamelCase						(       __lowercase						: Union[str, Any]						,__lowercase						: int						,__lowercase						: List[Any]=None	):
		'''simple docstring'''




		if not qid_list:
				A_		:				List[str]     =      len(a_	)
				return collections.OrderedDict(
				    [
				        ('exact', 1_00.0 * sum(exact_scores.values()	) / total),
				        ('f1', 1_00.0 * sum(fa_scores.values()	) / total),
				        ('total', total),
				    ]	)
		else:
				A_		:				List[str]     =      len(a_	)
				return collections.OrderedDict(
				    [
				        ('exact', 1_00.0 * sum(exact_scores[k] for k in qid_list	) / total),
				        ('f1', 1_00.0 * sum(fa_scores[k] for k in qid_list	) / total),
				        ('total', total),
				    ]	)




def 				UpperCamelCase						(       __lowercase						: int						,__lowercase						: Dict						,__lowercase						: Optional[int]	):
		'''simple docstring'''




		for k in new_eval:
				A_		:				List[Any]     =      new_eval[k]




def 				UpperCamelCase						(       __lowercase						: List[str]						,__lowercase						: Tuple						,__lowercase						: Dict						,__lowercase						: Optional[int]	):
		'''simple docstring'''




		plt.step(a_						,a_						,color='b'						,alpha=0.2						,where='post'	)
		plt.fill_between(a_						,a_						,step='post'						,alpha=0.2						,color='b'	)
		plt.xlabel('Recall'	)
		plt.ylabel('Precision'	)
		plt.xlim([0.0, 1.05]	)
		plt.ylim([0.0, 1.05]	)
		plt.title(a_	)
		plt.savefig(a_	)
		plt.clf()




def 				UpperCamelCase						(       __lowercase						: Optional[int]						,__lowercase						: str						,__lowercase						: List[str]						,__lowercase						: Union[str, Any]						,__lowercase						: List[Any]=None						,__lowercase						: Union[str, Any]=None	):
		'''simple docstring'''




		A_		:				Optional[int]     =      sorted(a_						,key=lambda __lowercase	: na_probs[k]	)
		A_		:				Optional[Any]     =      0.0
		A_		:				List[str]     =      1.0
		A_		:				str     =      0.0
		A_		:				Dict     =      [1.0]
		A_		:				List[Any]     =      [0.0]
		A_		:				List[str]     =      0.0
		for i, qid in enumerate(a_	):
				if qid_to_has_ans[qid]:
						true_pos += scores[qid]
				A_		:				Tuple     =      true_pos / float(i + 1	)
				A_		:				str     =      true_pos / float(a_	)
				if i == len(a_	) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
						# i.e., if we can put a threshold after this point
						avg_prec += cur_p * (cur_r - recalls[-1])
						precisions.append(a_	)
						recalls.append(a_	)
		if out_image:
				plot_pr_curve(a_						,a_						,a_						,a_	)
		return {"ap": 1_00.0 * avg_prec}




def 				UpperCamelCase						(       __lowercase						: int						,__lowercase						: str						,__lowercase						: Dict						,__lowercase						: int						,__lowercase						: List[Any]						,__lowercase						: Optional[int]	):
		'''simple docstring'''




		if out_image_dir and not os.path.exists(a_	):
				os.makedirs(a_	)
		A_		:				List[Any]     =      sum(1 for v in qid_to_has_ans.values() if v	)
		if num_true_pos == 0:
				return
		A_		:				Tuple     =      make_precision_recall_eval(
		    a_						,a_						,a_						,a_						,out_image=os.path.join(a_						,'pr_exact.png'	)						,title='Precision-Recall curve for Exact Match score'						,)
		A_		:				List[Any]     =      make_precision_recall_eval(
		    a_						,a_						,a_						,a_						,out_image=os.path.join(a_						,'pr_f1.png'	)						,title='Precision-Recall curve for F1 score'						,)
		A_		:				int     =      {k: float(a_	) for k, v in qid_to_has_ans.items()}
		A_		:				Union[str, Any]     =      make_precision_recall_eval(
		    a_						,a_						,a_						,a_						,out_image=os.path.join(a_						,'pr_oracle.png'	)						,title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'						,)
		merge_eval(a_						,a_						,'pr_exact'	)
		merge_eval(a_						,a_						,'pr_f1'	)
		merge_eval(a_						,a_						,'pr_oracle'	)




def 				UpperCamelCase						(       __lowercase						: List[str]						,__lowercase						: Tuple						,__lowercase						: Optional[Any]						,__lowercase						: str	):
		'''simple docstring'''




		if not qid_list:
				return
		A_		:				List[Any]     =      [na_probs[k] for k in qid_list]
		A_		:				Optional[int]     =      np.ones_like(a_	) / float(len(a_	)	)
		plt.hist(a_						,weights=a_						,bins=20						,range=(0.0, 1.0)	)
		plt.xlabel('Model probability of no-answer'	)
		plt.ylabel('Proportion of dataset'	)
		plt.title(f'''Histogram of no-answer probability: {name}'''	)
		plt.savefig(os.path.join(a_						,f'''na_prob_hist_{name}.png'''	)	)
		plt.clf()




def 				UpperCamelCase						(       __lowercase						: Tuple						,__lowercase						: Optional[int]						,__lowercase						: Optional[int]						,__lowercase						: Tuple	):
		'''simple docstring'''




		A_		:				Union[str, Any]     =      sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]	)
		A_		:				Union[str, Any]     =      num_no_ans
		A_		:				str     =      cur_score
		A_		:				Tuple     =      0.0
		A_		:				Optional[Any]     =      sorted(a_						,key=lambda __lowercase	: na_probs[k]	)
		for i, qid in enumerate(a_	):
				if qid not in scores:
						continue
				if qid_to_has_ans[qid]:
						A_		:				Dict     =      scores[qid]
				else:
						if preds[qid]:
								A_		:				Union[str, Any]     =      -1
						else:
								A_		:				Union[str, Any]     =      0
				cur_score += diff
				if cur_score > best_score:
						A_		:				Union[str, Any]     =      cur_score
						A_		:				Dict     =      na_probs[qid]
		return 1_00.0 * best_score / len(a_	), best_thresh




def 				UpperCamelCase						(       __lowercase						: Tuple						,__lowercase						: List[Any]						,__lowercase						: str						,__lowercase						: Optional[Any]						,__lowercase						: Optional[Any]						,__lowercase						: Optional[int]	):
		'''simple docstring'''




		A_   ,			A_		:				List[str]     =      find_best_thresh(a_						,a_						,a_						,a_	)
		A_   ,			A_		:				Tuple     =      find_best_thresh(a_						,a_						,a_						,a_	)
		A_		:				Tuple     =      best_exact
		A_		:				Optional[Any]     =      exact_thresh
		A_		:				List[str]     =      best_fa
		A_		:				List[Any]     =      fa_thresh




def 				UpperCamelCase						(       ):
		'''simple docstring'''




		with open(OPTS.data_file	) as f:
				A_		:				List[Any]     =      json.load(a_	)
				A_		:				Union[str, Any]     =      dataset_json['data']
		with open(OPTS.pred_file	) as f:
				A_		:				Optional[Any]     =      json.load(a_	)
		if OPTS.na_prob_file:
				with open(OPTS.na_prob_file	) as f:
						A_		:				Tuple     =      json.load(a_	)
		else:
				A_		:				Dict     =      {k: 0.0 for k in preds}
		A_		:				Union[str, Any]     =      make_qid_to_has_ans(a_	)  # maps qid to True/False
		A_		:				Optional[Any]     =      [k for k, v in qid_to_has_ans.items() if v]
		A_		:				List[str]     =      [k for k, v in qid_to_has_ans.items() if not v]
		A_   ,			A_		:				Optional[Any]     =      get_raw_scores(a_						,a_	)
		A_		:				Dict     =      apply_no_ans_threshold(a_						,a_						,a_						,OPTS.na_prob_thresh	)
		A_		:				int     =      apply_no_ans_threshold(a_						,a_						,a_						,OPTS.na_prob_thresh	)
		A_		:				Optional[int]     =      make_eval_dict(a_						,a_	)
		if has_ans_qids:
				A_		:				List[str]     =      make_eval_dict(a_						,a_						,qid_list=a_	)
				merge_eval(a_						,a_						,'HasAns'	)
		if no_ans_qids:
				A_		:				Optional[Any]     =      make_eval_dict(a_						,a_						,qid_list=a_	)
				merge_eval(a_						,a_						,'NoAns'	)
		if OPTS.na_prob_file:
				find_all_best_thresh(a_						,a_						,a_						,a_						,a_						,a_	)
		if OPTS.na_prob_file and OPTS.out_image_dir:
				run_precision_recall_analysis(a_						,a_						,a_						,a_						,a_						,OPTS.out_image_dir	)
				histogram_na_prob(a_						,a_						,OPTS.out_image_dir						,'hasAns'	)
				histogram_na_prob(a_						,a_						,OPTS.out_image_dir						,'noAns'	)
		if OPTS.out_file:
				with open(OPTS.out_file						,'w'	) as f:
						json.dump(a_						,a_	)
		else:
				print(json.dumps(a_						,indent=2	)	)


if __name__ == "__main__":
	_UpperCAmelCase						=		parse_args()
	if OPTS.out_image_dir:
		import matplotlib

		matplotlib.use("""Agg""")
		import matplotlib.pyplot as plt
	main()








import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE							:List[Any]							 =				logging.get_logger(__name__)

SCREAMING_SNAKE_CASE							:int							 =				{'vocab_file': 'spiece.model'}

SCREAMING_SNAKE_CASE							:Union[str, Any]							 =				{
    'vocab_file': {
        'google/bigbird-roberta-base': 'https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model',
        'google/bigbird-roberta-large': (
            'https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'
        ),
        'google/bigbird-base-trivia-itc': (
            'https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'
        ),
    }
}

SCREAMING_SNAKE_CASE							:int							 =				{
    'google/bigbird-roberta-base': 4096,
    'google/bigbird-roberta-large': 4096,
    'google/bigbird-base-trivia-itc': 4096,
}





class 							UpperCAmelCase      (  __SCREAMING_SNAKE_CASE  ):




	'''simple docstring'''





	snake_case_					    =     VOCAB_FILES_NAMES
	snake_case_					    =     PRETRAINED_VOCAB_FILES_MAP
	snake_case_					    =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	snake_case_					    =     ["input_ids", "attention_mask"]
	snake_case_					    =     []






	def __init__(       self					: Any							,A					: List[str]							,A					: str="<unk>"							,A					: int="<s>"							,A					: Union[str, Any]="</s>"							,A					: List[str]="<pad>"							,A					: int="[SEP]"							,A					: Optional[Any]="[MASK]"							,A					: Tuple="[CLS]"							,A					: Optional[Dict[str, Any]] = None							,**A					: Any							,):
					__A   			= AddedToken(A							,lstrip=A							,rstrip=A ) if isinstance(A							,A ) else bos_token
					__A   			= AddedToken(A							,lstrip=A							,rstrip=A ) if isinstance(A							,A ) else eos_token
					__A   			= AddedToken(A							,lstrip=A							,rstrip=A ) if isinstance(A							,A ) else unk_token
					__A   			= AddedToken(A							,lstrip=A							,rstrip=A ) if isinstance(A							,A ) else pad_token
					__A   			= AddedToken(A							,lstrip=A							,rstrip=A ) if isinstance(A							,A ) else cls_token
					__A   			= AddedToken(A							,lstrip=A							,rstrip=A ) if isinstance(A							,A ) else sep_token

					# Mask token behave like a normal word, i.e. include the space before it
					__A   			= AddedToken(A							,lstrip=A							,rstrip=A ) if isinstance(A							,A ) else mask_token

					__A   			= {} if sp_model_kwargs is None else sp_model_kwargs

					super().__init__(
					    bos_token=A							,eos_token=A							,unk_token=A							,pad_token=A							,sep_token=A							,mask_token=A							,cls_token=A							,sp_model_kwargs=self.sp_model_kwargs							,**A							,)

					__A   			= vocab_file

					__A   			= spm.SentencePieceProcessor(**self.sp_model_kwargs )
					self.sp_model.Load(A )






	@property
	def 				UpperCamelCase_  (       self					: List[str] ):
					return self.sp_model.get_piece_size()






	def 				UpperCamelCase_  (       self					: Optional[Any] ):
					__A   			= {self.convert_ids_to_tokens(A ): i for i in range(self.vocab_size )}
					vocab.update(self.added_tokens_encoder )
					return vocab






	def __getstate__(       self					: Optional[int] ):
					__A   			= self.__dict__.copy()
					__A   			= None
					return state






	def __setstate__(       self					: str							,A					: Optional[Any] ):
					__A   			= d

					# for backward compatibility
					if not hasattr(self							,"sp_model_kwargs" ):
									__A   			= {}

					__A   			= spm.SentencePieceProcessor(**self.sp_model_kwargs )
					self.sp_model.Load(self.vocab_file )






	def 				UpperCamelCase_  (       self					: Any							,A					: str ):
					return self.sp_model.encode(A							,out_type=A )






	def 				UpperCamelCase_  (       self					: List[str]							,A					: Tuple ):
					return self.sp_model.piece_to_id(A )






	def 				UpperCamelCase_  (       self					: List[Any]							,A					: Tuple ):
					__A   			= self.sp_model.IdToPiece(A )
					return token






	def 				UpperCamelCase_  (       self					: List[Any]							,A					: int ):
					__A   			= []
					__A   			= ""
					__A   			= False
					for token in tokens:
									# make sure that special tokens are not decoded using sentencepiece model
									if token in self.all_special_tokens:
													if not prev_is_special:
																	out_string += " "
													out_string += self.sp_model.decode(A ) + token
													__A   			= True
													__A   			= []
									else:
													current_sub_tokens.append(A )
													__A   			= False
					out_string += self.sp_model.decode(A )
					return out_string.strip()






	def 				UpperCamelCase_  (       self					: Tuple							,A					: List[int]							,A					: bool = False							,A					: bool = None							,A					: bool = True							,**A					: Union[str, Any]							,):
					__A   			= kwargs.pop("use_source_tokenizer"							,A )

					__A   			= self.convert_ids_to_tokens(A							,skip_special_tokens=A )

					# To avoid mixing byte-level and unicode for byte-level BPT
					# we need to build string separately for added tokens and byte-level tokens
					# cf. https://github.com/huggingface/transformers/issues/1133
					__A   			= []
					__A   			= []
					for token in filtered_tokens:
									if skip_special_tokens and token in self.all_special_ids:
													continue
									if token in self.added_tokens_encoder:
													if current_sub_text:
																	sub_texts.append(self.convert_tokens_to_string(A ) )
																	__A   			= []
													sub_texts.append(A )
									else:
													current_sub_text.append(A )
					if current_sub_text:
									sub_texts.append(self.convert_tokens_to_string(A ) )

					# Mimic the behavior of the Rust tokenizer:
					# No space before [MASK] and [SEP]
					if spaces_between_special_tokens:
									__A   			= re.sub(R" (\[(MASK|SEP)\])"							,R"\1"							," ".join(A ) )
					else:
									__A   			= "".join(A )

					__A   			= (
					    clean_up_tokenization_spaces
					    if clean_up_tokenization_spaces is not None
					    else self.clean_up_tokenization_spaces
					)
					if clean_up_tokenization_spaces:
									__A   			= self.clean_up_tokenization(A )
									return clean_text
					else:
									return text






	def 				UpperCamelCase_  (       self					: str							,A					: str							,A					: Optional[str] = None ):
					if not os.path.isdir(A ):
									logger.error(f'''Vocabulary path ({save_directory}) should be a directory''' )
									return
					__A   			= os.path.join(
					    A							,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )

					if os.path.abspath(self.vocab_file ) != os.path.abspath(A ) and os.path.isfile(self.vocab_file ):
									copyfile(self.vocab_file							,A )
					elif not os.path.isfile(self.vocab_file ):
									with open(A							,"wb" ) as fi:
													__A   			= self.sp_model.serialized_model_proto()
													fi.write(A )

					return (out_vocab_file,)






	def 				UpperCamelCase_  (       self					: Dict							,A					: List[int]							,A					: Optional[List[int]] = None ):
					if token_ids_a is None:
									return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
					__A   			= [self.cls_token_id]
					__A   			= [self.sep_token_id]
					return cls + token_ids_a + sep + token_ids_a + sep






	def 				UpperCamelCase_  (       self					: Optional[int]							,A					: List[int]							,A					: Optional[List[int]] = None							,A					: bool = False ):
					if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=A							,token_ids_a=A							,already_has_special_tokens=A )

					if token_ids_a is None:
									return [1] + ([0] * len(A )) + [1]
					return [1] + ([0] * len(A )) + [1] + ([0] * len(A )) + [1]







	def 				UpperCamelCase_  (       self					: Any							,A					: List[int]							,A					: Optional[List[int]] = None ):
					__A   			= [self.sep_token_id]
					__A   			= [self.cls_token_id]
					if token_ids_a is None:
									return len(cls + token_ids_a + sep ) * [0]
					return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]
