





'''simple docstring'''


import jax
import jax.numpy as jnp
from bigbird_flax import FlaxBigBirdForNaturalQuestions
from datasets import load_from_disk

from transformers import BigBirdTokenizerFast


_lowerCamelCase					:						Optional[int]					     =							{0: 'null', 1: 'short', 2: 'long', 3: 'yes', 4: 'no'}
_lowerCamelCase					:						int					     =							set(''.join(['‘', '’', '´', '`', '.', ',', '-', '"']))




def  __a			(  UpperCAmelCase							,      UpperCAmelCase=0							,      UpperCAmelCase=None     )   ->Any:



	"""simple docstring"""







	return [" ".join(x.split(""" """     )[begin:end]     ) for x in answers if len(x.split(""" """     )     ) > 1]




def  __a			(  UpperCAmelCase							,      UpperCAmelCase=False     )   ->int:



	"""simple docstring"""







	if make_sub_answers:
		# if answers are longer than one word, make sure a predictions is correct if it coresponds to the complete 1: or :-1 sub word
		# *e.g.* if the correct answer contains a prefix such as "the", or "a"
		A					=   (
		    given_answers + get_sub_answers(_SCREAMING_SNAKE_CASE							,      begin=1     ) + get_sub_answers(_SCREAMING_SNAKE_CASE							,      end=-1     )
		)
	A					=   []
	for answer in given_answers:
		A					=   answer.replace("""_"""							,      """ """     ).lower()
		A					=   """""".join(c if c not in PUNCTUATION_SET_TO_EXCLUDE else """ """ for c in alias     )
		answers.append(""" """.join(alias.split()     ).strip()     )
	return set(_SCREAMING_SNAKE_CASE     )




def  __a			(  UpperCAmelCase							,      UpperCAmelCase							,      UpperCAmelCase=1							,      UpperCAmelCase=100     )   ->Optional[int]:



	"""simple docstring"""







	A	,     A					=   jax.lax.top_k(_SCREAMING_SNAKE_CASE							,      _SCREAMING_SNAKE_CASE     )
	A	,     A					=   jax.lax.top_k(_SCREAMING_SNAKE_CASE							,      _SCREAMING_SNAKE_CASE     )

	A					=   best_end_idx[:, None] - best_start_idx[None, :]
	A					=   jnp.logical_or(widths < 0							,      widths > max_size     )
	A					=   (best_end_scores[:, None] + best_start_scores[None, :]) - (1E8 * mask)
	A					=   jnp.argmax(_SCREAMING_SNAKE_CASE     ).item()

	return best_start_idx[best_score % top_k], best_end_idx[best_score // top_k]




def  __a			(  UpperCAmelCase     )   ->Optional[Any]:



	"""simple docstring"""







	A					=   sample["""question"""]["""text"""]
	A					=   sample["""document"""]["""tokens"""]["""token"""]
	A					=   sample["""document"""]["""tokens"""]["""is_html"""]
	A					=   sample["""annotations"""]["""long_answer"""]
	A					=   sample["""annotations"""]["""short_answers"""]

	A					=   """ """.join([context[i] for i in range(len(_SCREAMING_SNAKE_CASE     )     ) if not is_html[i]]     )

	# 0 - No ; 1 - Yes
	for answer in sample["annotations"]["yes_no_answer"]:
		if answer == 0 or answer == 1:
			return {
			    "question": question,
			    "context": context_string,
			    "short": [],
			    "long": [],
			    "category": "no" if answer == 0 else "yes",
			}

	A					=   []
	for s in short_answers:
		short_targets.extend(s["""text"""]     )
	A					=   list(set(_SCREAMING_SNAKE_CASE     )     )

	A					=   []
	for s in long_answers:
		if s["start_token"] == -1:
			continue
		A					=   context[s["""start_token"""] : s["""end_token"""]]
		A					=   is_html[s["""start_token"""] : s["""end_token"""]]
		A					=   """ """.join([answer[i] for i in range(len(_SCREAMING_SNAKE_CASE     )     ) if not html[i]]     )
		if new_answer not in long_targets:
			long_targets.append(_SCREAMING_SNAKE_CASE     )

	A					=   """long_short""" if len(short_targets + long_targets     ) > 0 else """null"""

	return {
	    "question": question,
	    "context": context_string,
	    "short": short_targets,
	    "long": long_targets,
	    "category": category,
	}




def  __a			(  )   ->Union[str, Any]:



	"""simple docstring"""







	A					=   load_from_disk("""natural-questions-validation"""     )
	A					=   dataset.map(_SCREAMING_SNAKE_CASE     ).remove_columns(["""annotations""", """document""", """id"""]     )
	print(_SCREAMING_SNAKE_CASE     )

	A					=   dataset.filter(lambda UpperCAmelCase     : (len(x["""question"""]     ) + len(x["""context"""]     )) < 4 * 4096     )
	A					=   short_validation_dataset.filter(lambda UpperCAmelCase     : x["category"] != "null"     )
	short_validation_dataset

	A					=   """vasudevgupta/flax-bigbird-natural-questions"""
	A					=   FlaxBigBirdForNaturalQuestions.from_pretrained(_SCREAMING_SNAKE_CASE     )
	A					=   BigBirdTokenizerFast.from_pretrained(_SCREAMING_SNAKE_CASE     )

	@jax.jit
	def forward(*UpperCAmelCase							,      **UpperCAmelCase     ):
		A	,     A	,     A					=   model(*_SCREAMING_SNAKE_CASE							,      **_SCREAMING_SNAKE_CASE     )
		return start_logits, end_logits, jnp.argmax(_SCREAMING_SNAKE_CASE							,      axis=-1     )

	def evaluate(UpperCAmelCase     ):
		# encode question and context so that they are separated by a tokenizer.sep_token and cut at max_length
		A					=   tokenizer(
		    example["""question"""]							,      example["""context"""]							,      return_tensors="""np"""							,      max_length=4096							,      padding="""max_length"""							,      truncation=_SCREAMING_SNAKE_CASE							,      )

		A	,     A	,     A					=   forward(**_SCREAMING_SNAKE_CASE     )

		A					=   CATEGORY_MAPPING[category.item()]

		A					=   example["""long"""] + example["""short"""]
		if example["category"] in ["yes", "no", "null"]:
			A					=   [example["""category"""]]
		A					=   example["""category"""] != """null"""
		# Now target can be: "yes", "no", "null", "list of long & short answers"

		if predicted_category in ["yes", "no", "null"]:
			A					=   [predicted_category]
			A					=   example["""output"""] == example["""targets"""]
			A					=   predicted_category != """null"""
			return example

		A					=   38 if predicted_category == """short""" else 1024
		A	,     A					=   get_best_valid_start_end_idx(
		    start_scores[0]							,      end_scores[0]							,      top_k=8							,      max_size=_SCREAMING_SNAKE_CASE     )

		A					=   inputs["""input_ids"""][0].tolist()
		A					=   [tokenizer.decode(input_ids[start_score : end_score + 1]     )]

		A					=   expand_to_aliases(example["""targets"""]							,      make_sub_answers=_SCREAMING_SNAKE_CASE     )
		A					=   expand_to_aliases(example["""output"""]     )

		# some preprocessing to both prediction and answer
		A					=   {"""""".join(a.split()     ) for a in answers}
		A					=   {"""""".join(p.split()     ) for p in predictions}
		A					=   {s for s in predictions if s not in ["""``""", """''""", """`""", """'"""]}

		# if there is a common element, it's a exact match
		A					=   len(list(answers & predictions     )     ) > 0
		A					=   predicted_category != """null""" and len(_SCREAMING_SNAKE_CASE     ) > 0

		return example

	A					=   short_validation_dataset.map(_SCREAMING_SNAKE_CASE     )

	A					=   len(_SCREAMING_SNAKE_CASE     )
	A					=   len(short_validation_dataset.filter(lambda UpperCAmelCase     : x["match"] == 1     )     )
	print("""EM score:"""							,      (matched / total) * 100							,      """%"""     )


if __name__ == "__main__":
				main()





"""simple docstring"""
import argparse
import json
import re
import string
import sys
from collections import Counter







def     _a					(   _SCREAMING_SNAKE_CASE					)     ->					Any:

							def remove_articles(_SCREAMING_SNAKE_CASE					):
														return re.sub(r"""\b(a|an|the)\b"""					,			""" """					,			_SCREAMING_SNAKE_CASE					)

							def white_space_fix(_SCREAMING_SNAKE_CASE					):
														return " ".join(text.split()					)

							def remove_punc(_SCREAMING_SNAKE_CASE					):
														snake_case_     							=		set(string.punctuation					)
														return "".join(ch for ch in text if ch not in exclude					)

							def lower(_SCREAMING_SNAKE_CASE					):
														return text.lower()

							return white_space_fix(remove_articles(remove_punc(lower(_SCREAMING_SNAKE_CASE					)					)					)					)







def     _a					(   _SCREAMING_SNAKE_CASE					,			_SCREAMING_SNAKE_CASE					)     ->					Optional[int]:
							snake_case_     							=		normalize_answer(_SCREAMING_SNAKE_CASE					).split()
							snake_case_     							=		normalize_answer(_SCREAMING_SNAKE_CASE					).split()
							snake_case_     							=		Counter(_SCREAMING_SNAKE_CASE					) & Counter(_SCREAMING_SNAKE_CASE					)
							snake_case_     							=		sum(common.values()					)
							if num_same == 0:
														return 0
							snake_case_     							=		1.0 * num_same / len(_SCREAMING_SNAKE_CASE					)
							snake_case_     							=		1.0 * num_same / len(_SCREAMING_SNAKE_CASE					)
							snake_case_     							=		(2 * precision * recall) / (precision + recall)
							return fa







def     _a					(   _SCREAMING_SNAKE_CASE					,			_SCREAMING_SNAKE_CASE					)     ->					Union[str, Any]:
							return normalize_answer(_SCREAMING_SNAKE_CASE					) == normalize_answer(_SCREAMING_SNAKE_CASE					)







def     _a					(   _SCREAMING_SNAKE_CASE					,			_SCREAMING_SNAKE_CASE					,			_SCREAMING_SNAKE_CASE					)     ->					Optional[int]:
							snake_case_     							=		[]
							for ground_truth in ground_truths:
														snake_case_     							=		metric_fn(_SCREAMING_SNAKE_CASE					,			_SCREAMING_SNAKE_CASE					)
														scores_for_ground_truths.append(_SCREAMING_SNAKE_CASE					)
							return max(_SCREAMING_SNAKE_CASE					)







def     _a					(   _SCREAMING_SNAKE_CASE					,			_SCREAMING_SNAKE_CASE					)     ->					Optional[int]:
							snake_case_     							=		snake_case_     							=		snake_case_     							=		0
							for article in dataset:
														for paragraph in article["paragraphs"]:
																					for qa in paragraph["qas"]:
																												total += 1
																												if qa["id"] not in predictions:
																																			snake_case_     							=		"""Unanswered question """ + qa["""id"""] + """ will receive score 0."""
																																			print(_SCREAMING_SNAKE_CASE					,			file=sys.stderr					)
																																			continue
																												snake_case_     							=		[x["""text"""] for x in qa["""answers"""]]
																												snake_case_     							=		predictions[qa["""id"""]]
																												exact_match += metric_max_over_ground_truths(_SCREAMING_SNAKE_CASE					,			_SCREAMING_SNAKE_CASE					,			_SCREAMING_SNAKE_CASE					)
																												fa += metric_max_over_ground_truths(_SCREAMING_SNAKE_CASE					,			_SCREAMING_SNAKE_CASE					,			_SCREAMING_SNAKE_CASE					)

							snake_case_     							=		100.0 * exact_match / total
							snake_case_     							=		100.0 * fa / total

							return {"exact_match": exact_match, "f1": fa}


if __name__ == "__main__":
				__SCREAMING_SNAKE_CASE				:     Tuple		=     '1.1'
				__SCREAMING_SNAKE_CASE				:     Union[str, Any]		=     argparse.ArgumentParser(description='Evaluation for SQuAD ' + expected_version)
				parser.add_argument('dataset_file', help='Dataset file')
				parser.add_argument('prediction_file', help='Prediction File')
				__SCREAMING_SNAKE_CASE				:     Union[str, Any]		=     parser.parse_args()
				with open(args.dataset_file) as dataset_file:
								__SCREAMING_SNAKE_CASE				:     Optional[Any]		=     json.load(dataset_file)
								if dataset_json["version"] != expected_version:
												print(
												    'Evaluation expects v-' + expected_version + ', but got dataset with v-' + dataset_json['version'],
												    file=sys.stderr,
												)
								__SCREAMING_SNAKE_CASE				:     str		=     dataset_json['data']
				with open(args.prediction_file) as prediction_file:
								__SCREAMING_SNAKE_CASE				:     int		=     json.load(prediction_file)
				print(json.dumps(evaluate(dataset, predictions)))



