




import jax
import jax.numpy as jnp
from bigbird_flax import FlaxBigBirdForNaturalQuestions
from datasets import load_from_disk

from transformers import BigBirdTokenizerFast


UpperCAmelCase_    ={0: """null""", 1: """short""", 2: """long""", 3: """yes""", 4: """no"""}
UpperCAmelCase_    =set("""""".join(["""‘""", """’""", """´""", """`""", """.""", """,""", """-""", """\""""]))



def    UpperCAmelCase							(  _snake_case    ,			_snake_case=0    ,			_snake_case=None				):
							return [" ".join(x.split(''' '''				)[begin:end]				) for x in answers if len(x.split(''' '''				)				) > 1]



def    UpperCAmelCase							(  _snake_case    ,			_snake_case=False				):
							if make_sub_answers:
														# if answers are longer than one word, make sure a predictions is correct if it coresponds to the complete 1: or :-1 sub word
														# *e.g.* if the correct answer contains a prefix such as "the", or "a"
														lowerCAmelCase =		(
														    given_answers + get_sub_answers(_snake_case    ,			begin=1				) + get_sub_answers(_snake_case    ,			end=-1				)
														)
							lowerCAmelCase =		[]
							for answer in given_answers:
														lowerCAmelCase =		answer.replace('''_'''    ,			''' '''				).lower()
														lowerCAmelCase =		''''''.join(c if c not in PUNCTUATION_SET_TO_EXCLUDE else ''' ''' for c in alias				)
														answers.append(''' '''.join(alias.split()				).strip()				)
							return set(_snake_case				)



def    UpperCAmelCase							(  _snake_case    ,			_snake_case    ,			_snake_case=1    ,			_snake_case=100				):
							lowerCAmelCase ,  lowerCAmelCase =		jax.lax.top_k(_snake_case    ,			_snake_case				)
							lowerCAmelCase ,  lowerCAmelCase =		jax.lax.top_k(_snake_case    ,			_snake_case				)

							lowerCAmelCase =		best_end_idx[:, None] - best_start_idx[None, :]
							lowerCAmelCase =		jnp.logical_or(widths < 0    ,			widths > max_size				)
							lowerCAmelCase =		(best_end_scores[:, None] + best_start_scores[None, :]) - (1E8 * mask)
							lowerCAmelCase =		jnp.argmax(_snake_case				).item()

							return best_start_idx[best_score % top_k], best_end_idx[best_score // top_k]



def    UpperCAmelCase							(  _snake_case				):
							lowerCAmelCase =		sample['''question''']['''text''']
							lowerCAmelCase =		sample['''document''']['''tokens''']['''token''']
							lowerCAmelCase =		sample['''document''']['''tokens''']['''is_html''']
							lowerCAmelCase =		sample['''annotations''']['''long_answer''']
							lowerCAmelCase =		sample['''annotations''']['''short_answers''']

							lowerCAmelCase =		''' '''.join([context[i] for i in range(len(_snake_case				)				) if not is_html[i]]				)

							# 0 - No ; 1 - Yes
							for answer in sample["annotations"]["yes_no_answer"]:
														if answer == 0 or answer == 1:
																					return {
																					    "question": question,
																					    "context": context_string,
																					    "short": [],
																					    "long": [],
																					    "category": "no" if answer == 0 else "yes",
																					}

							lowerCAmelCase =		[]
							for s in short_answers:
														short_targets.extend(s['''text''']				)
							lowerCAmelCase =		list(set(_snake_case				)				)

							lowerCAmelCase =		[]
							for s in long_answers:
														if s["start_token"] == -1:
																					continue
														lowerCAmelCase =		context[s['''start_token'''] : s['''end_token''']]
														lowerCAmelCase =		is_html[s['''start_token'''] : s['''end_token''']]
														lowerCAmelCase =		''' '''.join([answer[i] for i in range(len(_snake_case				)				) if not html[i]]				)
														if new_answer not in long_targets:
																					long_targets.append(_snake_case				)

							lowerCAmelCase =		'''long_short''' if len(short_targets + long_targets				) > 0 else '''null'''

							return {
							    "question": question,
							    "context": context_string,
							    "short": short_targets,
							    "long": long_targets,
							    "category": category,
							}



def    UpperCAmelCase							(  ):
							lowerCAmelCase =		load_from_disk('''natural-questions-validation'''				)
							lowerCAmelCase =		dataset.map(_snake_case				).remove_columns(['''annotations''', '''document''', '''id''']				)
							print(_snake_case				)

							lowerCAmelCase =		dataset.filter(lambda _snake_case				: (len(x['''question''']				) + len(x['''context''']				)) < 4 * 4096				)
							lowerCAmelCase =		short_validation_dataset.filter(lambda _snake_case				: x["category"] != "null"				)
							short_validation_dataset

							lowerCAmelCase =		'''vasudevgupta/flax-bigbird-natural-questions'''
							lowerCAmelCase =		FlaxBigBirdForNaturalQuestions.from_pretrained(_snake_case				)
							lowerCAmelCase =		BigBirdTokenizerFast.from_pretrained(_snake_case				)

							@jax.jit
							def forward(*_snake_case    ,			**_snake_case				):
														lowerCAmelCase ,  lowerCAmelCase ,  lowerCAmelCase =		model(*_snake_case    ,			**_snake_case				)
														return start_logits, end_logits, jnp.argmax(_snake_case    ,			axis=-1				)

							def evaluate(_snake_case				):
														# encode question and context so that they are separated by a tokenizer.sep_token and cut at max_length
														lowerCAmelCase =		tokenizer(
														    example['''question''']    ,			example['''context''']    ,			return_tensors='''np'''    ,			max_length=4096    ,			padding='''max_length'''    ,			truncation=_snake_case    ,			)

														lowerCAmelCase ,  lowerCAmelCase ,  lowerCAmelCase =		forward(**_snake_case				)

														lowerCAmelCase =		CATEGORY_MAPPING[category.item()]

														lowerCAmelCase =		example['''long'''] + example['''short''']
														if example["category"] in ["yes", "no", "null"]:
																					lowerCAmelCase =		[example['''category''']]
														lowerCAmelCase =		example['''category'''] != '''null'''
														# Now target can be: "yes", "no", "null", "list of long & short answers"

														if predicted_category in ["yes", "no", "null"]:
																					lowerCAmelCase =		[predicted_category]
																					lowerCAmelCase =		example['''output'''] == example['''targets''']
																					lowerCAmelCase =		predicted_category != '''null'''
																					return example

														lowerCAmelCase =		38 if predicted_category == '''short''' else 1024
														lowerCAmelCase ,  lowerCAmelCase =		get_best_valid_start_end_idx(
														    start_scores[0]    ,			end_scores[0]    ,			top_k=8    ,			max_size=_snake_case				)

														lowerCAmelCase =		inputs['''input_ids'''][0].tolist()
														lowerCAmelCase =		[tokenizer.decode(input_ids[start_score : end_score + 1]				)]

														lowerCAmelCase =		expand_to_aliases(example['''targets''']    ,			make_sub_answers=_snake_case				)
														lowerCAmelCase =		expand_to_aliases(example['''output''']				)

														# some preprocessing to both prediction and answer
														lowerCAmelCase =		{''''''.join(a.split()				) for a in answers}
														lowerCAmelCase =		{''''''.join(p.split()				) for p in predictions}
														lowerCAmelCase =		{s for s in predictions if s not in ['''``''', '''\'\'''', '''`''', '''\'''']}

														# if there is a common element, it's a exact match
														lowerCAmelCase =		len(list(answers & predictions				)				) > 0
														lowerCAmelCase =		predicted_category != '''null''' and len(_snake_case				) > 0

														return example

							lowerCAmelCase =		short_validation_dataset.map(_snake_case				)

							lowerCAmelCase =		len(_snake_case				)
							lowerCAmelCase =		len(short_validation_dataset.filter(lambda _snake_case				: x["match"] == 1				)				)
							print('''EM score:'''    ,			(matched / total) * 100    ,			'''%'''				)


if __name__ == "__main__":
		main()









from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


UpperCAmelCase_    ={"""configuration_vilt""": ["""VILT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """ViltConfig"""]}

try:
		if not is_vision_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCAmelCase_    =["""ViltFeatureExtractor"""]
		UpperCAmelCase_    =["""ViltImageProcessor"""]
		UpperCAmelCase_    =["""ViltProcessor"""]

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCAmelCase_    =[
		    """VILT_PRETRAINED_MODEL_ARCHIVE_LIST""",
		    """ViltForImageAndTextRetrieval""",
		    """ViltForImagesAndTextClassification""",
		    """ViltForTokenClassification""",
		    """ViltForMaskedLM""",
		    """ViltForQuestionAnswering""",
		    """ViltLayer""",
		    """ViltModel""",
		    """ViltPreTrainedModel""",
		]


if TYPE_CHECKING:
		from .configuration_vilt import VILT_PRETRAINED_CONFIG_ARCHIVE_MAP, ViltConfig

		try:
				if not is_vision_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .feature_extraction_vilt import ViltFeatureExtractor
				from .image_processing_vilt import ViltImageProcessor
				from .processing_vilt import ViltProcessor

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_vilt import (
				    VILT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    ViltForImageAndTextRetrieval,
				    ViltForImagesAndTextClassification,
				    ViltForMaskedLM,
				    ViltForQuestionAnswering,
				    ViltForTokenClassification,
				    ViltLayer,
				    ViltModel,
				    ViltPreTrainedModel,
				)


else:
		import sys

		UpperCAmelCase_    =_LazyModule(__name__, globals()["""__file__"""], _import_structure)



