
'''simple docstring'''
import datasets

from .evaluate import (
    apply_no_ans_threshold,
    find_all_best_thresh,
    get_raw_scores,
    make_eval_dict,
    make_qid_to_has_ans,
    merge_eval,
)


A__ :		List[Any]					='\\n@inproceedings{Rajpurkar2016SQuAD10,\n  title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text},\n  author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang},\n  booktitle={EMNLP},\n  year={2016}\n}\n'

A__ :		Optional[Any]					='\nThis metric wrap the official scoring script for version 2 of the Stanford Question\nAnswering Dataset (SQuAD).\n\nStanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by\ncrowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span,\nfrom the corresponding reading passage, or the question might be unanswerable.\n\nSQuAD2.0 combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions\nwritten adversarially by crowdworkers to look similar to answerable ones.\nTo do well on SQuAD2.0, systems must not only answer questions when possible, but also\ndetermine when no answer is supported by the paragraph and abstain from answering.\n'

A__ :		Tuple					='\nComputes SQuAD v2 scores (F1 and EM).\nArgs:\n    predictions: List of triple for question-answers to score with the following elements:\n        - the question-answer \'id\' field as given in the references (see below)\n        - the text of the answer\n        - the probability that the question has no answer\n    references: List of question-answers dictionaries with the following key-values:\n            - \'id\': id of the question-answer pair (see above),\n            - \'answers\': a list of Dict {\'text\': text of the answer as a string}\n    no_answer_threshold: float\n        Probability threshold to decide that a question has no answer.\nReturns:\n    \'exact\': Exact match (the normalized answer exactly match the gold answer)\n    \'f1\': The F-score of predicted tokens versus the gold answer\n    \'total\': Number of score considered\n    \'HasAns_exact\': Exact match (the normalized answer exactly match the gold answer)\n    \'HasAns_f1\': The F-score of predicted tokens versus the gold answer\n    \'HasAns_total\': Number of score considered\n    \'NoAns_exact\': Exact match (the normalized answer exactly match the gold answer)\n    \'NoAns_f1\': The F-score of predicted tokens versus the gold answer\n    \'NoAns_total\': Number of score considered\n    \'best_exact\': Best exact match (with varying threshold)\n    \'best_exact_thresh\': No-answer probability threshold associated to the best exact match\n    \'best_f1\': Best F1 (with varying threshold)\n    \'best_f1_thresh\': No-answer probability threshold associated to the best F1\nExamples:\n\n    >>> predictions = [{\'prediction_text\': \'1976\', \'id\': \'56e10a3be3433e1400422b22\', \'no_answer_probability\': 0.}]\n    >>> references = [{\'answers\': {\'answer_start\': [97], \'text\': [\'1976\']}, \'id\': \'56e10a3be3433e1400422b22\'}]\n    >>> squad_v2_metric = datasets.load_metric("squad_v2")\n    >>> results = squad_v2_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact\': 100.0, \'f1\': 100.0, \'total\': 1, \'HasAns_exact\': 100.0, \'HasAns_f1\': 100.0, \'HasAns_total\': 1, \'best_exact\': 100.0, \'best_exact_thresh\': 0.0, \'best_f1\': 100.0, \'best_f1_thresh\': 0.0}\n'







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION     ,    _KWARGS_DESCRIPTION      )
class 			__A					(			datasets.Metric      ):
 def 		lowercase_( self      :     str							):



   """simple docstring"""





   return datasets.MetricInfo(
       description=_DESCRIPTION					, citation=_CITATION					, inputs_description=_KWARGS_DESCRIPTION					, features=datasets.Features(
           {
               """predictions""": {
                   """id""": datasets.Value("""string"""							),
                   """prediction_text""": datasets.Value("""string"""							),
                   """no_answer_probability""": datasets.Value("""float32"""							),
               },
               """references""": {
                   """id""": datasets.Value("""string"""							),
                   """answers""": datasets.features.Sequence(
                       {"""text""": datasets.Value("""string"""							), """answer_start""": datasets.Value("""int32"""							)}							),
               },
           }							)					, codebase_urls=["""https://rajpurkar.github.io/SQuAD-explorer/"""]					, reference_urls=["""https://rajpurkar.github.io/SQuAD-explorer/"""]					, )






 def 		lowercase_( self      :     Tuple					, lowerCamelCase      :     str					, lowerCamelCase      :     int					, lowerCamelCase      :     Tuple=1.0							):



   """simple docstring"""





   __A		:  Tuple       				=		{p["""id"""]: p["""no_answer_probability"""] for p in predictions}
   __A		:  int       				=		[{"""paragraphs""": [{"""qas""": references}]}]
   __A		:  List[str]       				=		{p["""id"""]: p["""prediction_text"""] for p in predictions}

   __A		:  List[Any]       				=		make_qid_to_has_ans(_lowerCAmelCase							)  # maps qid to True/False
   __A		:  int       				=		[k for k, v in qid_to_has_ans.items() if v]
   __A		:  Optional[int]       				=		[k for k, v in qid_to_has_ans.items() if not v]

   __A				,       __A		:  Optional[int]       				=		get_raw_scores(_lowerCAmelCase					, _lowerCAmelCase							)
   __A		:  int       				=		apply_no_ans_threshold(_lowerCAmelCase					, _lowerCAmelCase					, _lowerCAmelCase					, _lowerCAmelCase							)
   __A		:  Union[str, Any]       				=		apply_no_ans_threshold(_lowerCAmelCase					, _lowerCAmelCase					, _lowerCAmelCase					, _lowerCAmelCase							)
   __A		:  Any       				=		make_eval_dict(_lowerCAmelCase					, _lowerCAmelCase							)

   if has_ans_qids:
     __A		:  Any       				=		make_eval_dict(_lowerCAmelCase					, _lowerCAmelCase					, qid_list=_lowerCAmelCase							)
     merge_eval(_lowerCAmelCase					, _lowerCAmelCase					, """HasAns"""							)
   if no_ans_qids:
     __A		:  List[Any]       				=		make_eval_dict(_lowerCAmelCase					, _lowerCAmelCase					, qid_list=_lowerCAmelCase							)
     merge_eval(_lowerCAmelCase					, _lowerCAmelCase					, """NoAns"""							)
   find_all_best_thresh(_lowerCAmelCase					, _lowerCAmelCase					, _lowerCAmelCase					, _lowerCAmelCase					, _lowerCAmelCase					, _lowerCAmelCase							)
   return dict(_lowerCAmelCase							)










'''simple docstring'''


import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


_lowerCamelCase					:						Dict					     =							logging.get_logger(__name__)

_lowerCamelCase					:						Dict					     =							{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

_lowerCamelCase					:						str					     =							{
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

_lowerCamelCase					:						Union[str, Any]					     =							{
    'openai-gpt': 512,
}




def  __a			(  UpperCAmelCase     )   ->Union[str, Any]:



	"""simple docstring"""







	A					=   text.strip()
	if not text:
		return []
	A					=   text.split()
	return tokens



class 				__UpperCAmelCase	(							A__    ):







		'''simple docstring'''







		def __init__(self    :   Optional[int]   ,   _lowerCAmelCase    :   int=True   ,   _lowerCAmelCase    :   List[str]=None   ,   _lowerCAmelCase    :   str=True   ,   _lowerCAmelCase    :   int=None   ,   _lowerCAmelCase    :   Optional[Any]=True   ,   ):
			if never_split is None:
				A					=   []
			A					=   do_lower_case
			A					=   set(_lowerCAmelCase   )
			A					=   tokenize_chinese_chars
			A					=   strip_accents
			A					=   do_split_on_punc







		def 							A  (self    :   List[str]   ,   _lowerCAmelCase    :   Any   ,   _lowerCAmelCase    :   Any=None   ):
			A					=   self.never_split.union(set(_lowerCAmelCase   )   ) if never_split else self.never_split
			A					=   self._clean_text(_lowerCAmelCase   )

			# This was added on November 1st, 2018 for the multilingual and Chinese
			# models. This is also applied to the English models now, but it doesn't
			# matter since the English models were not trained on any Chinese data
			# and generally don't have any Chinese data in them (there are Chinese
			# characters in the vocabulary because Wikipedia does have some Chinese
			# words in the English Wikipedia.).
			if self.tokenize_chinese_chars:
				A					=   self._tokenize_chinese_chars(_lowerCAmelCase   )
			# prevents treating the same character with different unicode codepoints as different characters
			A					=   unicodedata.normalize("""NFC"""   ,   _lowerCAmelCase   )
			A					=   whitespace_tokenize(_lowerCAmelCase   )
			A					=   []
			for token in orig_tokens:
				if token not in never_split:
					if self.do_lower_case:
						A					=   token.lower()
						if self.strip_accents is not False:
							A					=   self._run_strip_accents(_lowerCAmelCase   )
					elif self.strip_accents:
						A					=   self._run_strip_accents(_lowerCAmelCase   )
				split_tokens.extend(self._run_split_on_punc(_lowerCAmelCase   ,   _lowerCAmelCase   )   )

			A					=   whitespace_tokenize(""" """.join(_lowerCAmelCase   )   )
			return output_tokens







		def 							A  (self    :   Optional[Any]   ,   _lowerCAmelCase    :   Optional[int]   ):
			A					=   unicodedata.normalize("""NFD"""   ,   _lowerCAmelCase   )
			A					=   []
			for char in text:
				A					=   unicodedata.category(_lowerCAmelCase   )
				if cat == "Mn":
					continue
				output.append(_lowerCAmelCase   )
			return "".join(_lowerCAmelCase   )







		def 							A  (self    :   str   ,   _lowerCAmelCase    :   Dict   ,   _lowerCAmelCase    :   List[str]=None   ):
			if not self.do_split_on_punc or (never_split is not None and text in never_split):
				return [text]
			A					=   list(_lowerCAmelCase   )
			A					=   0
			A					=   True
			A					=   []
			while i < len(_lowerCAmelCase   ):
				A					=   chars[i]
				if _is_punctuation(_lowerCAmelCase   ):
					output.append([char]   )
					A					=   True
				else:
					if start_new_word:
						output.append([]   )
					A					=   False
					output[-1].append(_lowerCAmelCase   )
				i += 1

			return ["".join(_lowerCAmelCase   ) for x in output]







		def 							A  (self    :   int   ,   _lowerCAmelCase    :   Any   ):
			A					=   []
			for char in text:
				A					=   ord(_lowerCAmelCase   )
				if self._is_chinese_char(_lowerCAmelCase   ):
					output.append(""" """   )
					output.append(_lowerCAmelCase   )
					output.append(""" """   )
				else:
					output.append(_lowerCAmelCase   )
			return "".join(_lowerCAmelCase   )







		def 							A  (self    :   Dict   ,   _lowerCAmelCase    :   Any   ):
			# This defines a "chinese character" as anything in the CJK Unicode block:
			#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
			#
			# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
			# despite its name. The modern Korean Hangul alphabet is a different block,
			# as is Japanese Hiragana and Katakana. Those alphabets are used to write
			# space-separated words, so they are not treated specially and handled
			# like the all of the other languages.
			if (
			    (cp >= 0X4_e00 and cp <= 0X9_fff)
			    or (cp >= 0X3_400 and cp <= 0X4_dbf)  #
			    or (cp >= 0X20_000 and cp <= 0X2a_6df)  #
			    or (cp >= 0X2a_700 and cp <= 0X2b_73f)  #
			    or (cp >= 0X2b_740 and cp <= 0X2b_81f)  #
			    or (cp >= 0X2b_820 and cp <= 0X2c_eaf)  #
			    or (cp >= 0Xf_900 and cp <= 0Xf_aff)
			    or (cp >= 0X2f_800 and cp <= 0X2f_a1f)  #
			):  #
				return True

			return False







		def 							A  (self    :   List[str]   ,   _lowerCAmelCase    :   Any   ):
			A					=   []
			for char in text:
				A					=   ord(_lowerCAmelCase   )
				if cp == 0 or cp == 0Xf_ffd or _is_control(_lowerCAmelCase   ):
					continue
				if _is_whitespace(_lowerCAmelCase   ):
					output.append(""" """   )
				else:
					output.append(_lowerCAmelCase   )
			return "".join(_lowerCAmelCase   )






def  __a			(  UpperCAmelCase     )   ->Union[str, Any]:



	"""simple docstring"""







	A					=   set()
	A					=   word[0]
	for char in word[1:]:
		pairs.add((prev_char, char)     )
		A					=   char
	return pairs




def  __a			(  UpperCAmelCase     )   ->List[str]:



	"""simple docstring"""







	A					=   text.replace("""—"""							,      """-"""     )
	A					=   text.replace("""–"""							,      """-"""     )
	A					=   text.replace("""―"""							,      """-"""     )
	A					=   text.replace("""…"""							,      """..."""     )
	A					=   text.replace("""´"""							,      """'"""     )
	A					=   re.sub(R"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"""							,      R""" \1 """							,      UpperCAmelCase     )
	A					=   re.sub(R"""\s*\n\s*"""							,      """ \n """							,      UpperCAmelCase     )
	A					=   re.sub(R"""[^\S\n]+"""							,      """ """							,      UpperCAmelCase     )
	return text.strip()



class 				__UpperCAmelCase	(							A__    ):







		'''simple docstring'''







		__lowerCAmelCase								=     VOCAB_FILES_NAMES
		__lowerCAmelCase								=     PRETRAINED_VOCAB_FILES_MAP
		__lowerCAmelCase								=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		__lowerCAmelCase								=     ['''input_ids''', '''attention_mask''']







		def __init__(self    :   Dict   ,   _lowerCAmelCase    :   Optional[int]   ,   _lowerCAmelCase    :   List[Any]   ,   _lowerCAmelCase    :   Dict="<unk>"   ,   **_lowerCAmelCase    :   Dict   ):
			super().__init__(unk_token=_lowerCAmelCase   ,   **_lowerCAmelCase   )

			try:
				import ftfy
				from spacy.lang.en import English

				A					=   English()
				A					=   _nlp.tokenizer
				A					=   ftfy.fix_text
			except ImportError:
				logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""   )
				A					=   BasicTokenizer(do_lower_case=_lowerCAmelCase   )
				A					=   None

			with open(_lowerCAmelCase   ,   encoding="""utf-8"""   ) as vocab_handle:
				A					=   json.load(_lowerCAmelCase   )
			A					=   {v: k for k, v in self.encoder.items()}
			with open(_lowerCAmelCase   ,   encoding="""utf-8"""   ) as merges_handle:
				A					=   merges_handle.read().split("""\n"""   )[1:-1]
			A					=   [tuple(merge.split()   ) for merge in merges]
			A					=   dict(zip(_lowerCAmelCase   ,   range(len(_lowerCAmelCase   )   )   )   )
			A					=   {}







		@property
		def 							A  (self    :   Dict   ):
			return True







		@property
		def 							A  (self    :   Dict   ):
			return len(self.encoder   )







		def 							A  (self    :   str   ):
			return dict(self.encoder   ,   **self.added_tokens_encoder   )







		def 							A  (self    :   Optional[Any]   ,   _lowerCAmelCase    :   List[Any]   ):
			A					=   tuple(token[:-1]   ) + (token[-1] + """</w>""",)
			if token in self.cache:
				return self.cache[token]
			A					=   get_pairs(_lowerCAmelCase   )

			if not pairs:
				return token + "</w>"

			while True:
				A					=   min(_lowerCAmelCase   ,   key=lambda _lowerCAmelCase   : self.bpe_ranks.get(_lowerCAmelCase   ,   float("""inf"""   )   )   )
				if bigram not in self.bpe_ranks:
					break
				A	,     A					=   bigram
				A					=   []
				A					=   0
				while i < len(_lowerCAmelCase   ):
					try:
						A					=   word.index(_lowerCAmelCase   ,   _lowerCAmelCase   )
					except ValueError:
						new_word.extend(word[i:]   )
						break
					else:
						new_word.extend(word[i:j]   )
						A					=   j

					if word[i] == first and i < len(_lowerCAmelCase   ) - 1 and word[i + 1] == second:
						new_word.append(first + second   )
						i += 2
					else:
						new_word.append(word[i]   )
						i += 1
				A					=   tuple(_lowerCAmelCase   )
				A					=   new_word
				if len(_lowerCAmelCase   ) == 1:
					break
				else:
					A					=   get_pairs(_lowerCAmelCase   )
			A					=   """ """.join(_lowerCAmelCase   )
			if word == "\n  </w>":
				A					=   """\n</w>"""
			A					=   word
			return word







		def 							A  (self    :   List[str]   ,   _lowerCAmelCase    :   Optional[int]   ):
			A					=   []
			if self.fix_text is None:
				# Using BERT's BasicTokenizer
				A					=   self.nlp.tokenize(_lowerCAmelCase   )
				for token in text:
					split_tokens.extend(list(self.bpe(_lowerCAmelCase   ).split(""" """   )   )   )
			else:
				# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
				A					=   self.nlp(text_standardize(self.fix_text(_lowerCAmelCase   )   )   )
				for token in text:
					split_tokens.extend(list(self.bpe(token.text.lower()   ).split(""" """   )   )   )
			return split_tokens







		def 							A  (self    :   int   ,   _lowerCAmelCase    :   Dict   ):
			return self.encoder.get(_lowerCAmelCase   ,   self.encoder.get(self.unk_token   )   )







		def 							A  (self    :   Dict   ,   _lowerCAmelCase    :   Tuple   ):
			return self.decoder.get(_lowerCAmelCase   ,   self.unk_token   )







		def 							A  (self    :   Optional[int]   ,   _lowerCAmelCase    :   int   ):
			A					=   """""".join(_lowerCAmelCase   ).replace("""</w>"""   ,   """ """   ).strip()
			return out_string







		def 							A  (self    :   str   ,   _lowerCAmelCase    :   str   ,   _lowerCAmelCase    :   Optional[str] = None   ):
			if not os.path.isdir(_lowerCAmelCase   ):
				logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
				return
			A					=   os.path.join(
			    _lowerCAmelCase   ,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )
			A					=   os.path.join(
			    _lowerCAmelCase   ,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]   )

			with open(_lowerCAmelCase   ,   """w"""   ,   encoding="""utf-8"""   ) as f:
				f.write(json.dumps(self.encoder   ,   indent=2   ,   sort_keys=_lowerCAmelCase   ,   ensure_ascii=_lowerCAmelCase   ) + """\n"""   )

			A					=   0
			with open(_lowerCAmelCase   ,   """w"""   ,   encoding="""utf-8"""   ) as writer:
				writer.write("""#version: 0.2\n"""   )
				for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,   key=lambda _lowerCAmelCase   : kv[1]   ):
					if index != token_index:
						logger.warning(
						    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
						    """ Please check that the tokenizer is not corrupted!"""   )
						A					=   token_index
					writer.write(""" """.join(_lowerCAmelCase   ) + """\n"""   )
					index += 1

			return vocab_file, merge_file

