




'''simple docstring'''




import tempfile
import unittest

import numpy as np

import transformers
from transformers import GPTaConfig, GPTaTokenizer, is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
							import jax
							import jax.numpy as jnp

							from transformers.modeling_flax_pytorch_utils import (
							    convert_pytorch_state_dict_to_flax,
							    load_flax_weights_in_pytorch_model,
							)
							from transformers.models.gpta.modeling_flax_gpta import FlaxGPTaLMHeadModel, FlaxGPTaModel

if is_torch_available():
							import torch






class      UpperCamelCase_  :







						"""simple docstring"""


						def __init__(  self   :							List[Any]				,			_lowerCamelCase   :							List[Any]				,			_lowerCamelCase   :							Union[str, Any]=14				,			_lowerCamelCase   :							List[Any]=7				,			_lowerCamelCase   :							List[Any]=True				,			_lowerCamelCase   :							Union[str, Any]=True				,			_lowerCamelCase   :							Dict=False				,			_lowerCamelCase   :							Optional[Any]=True				,			_lowerCamelCase   :							List[str]=99				,			_lowerCamelCase   :							Any=32				,			_lowerCamelCase   :							Tuple=5				,			_lowerCamelCase   :							List[str]=4				,			_lowerCamelCase   :							Optional[int]=37				,			_lowerCamelCase   :							Optional[Any]="gelu"				,			_lowerCamelCase   :							str=0.1				,			_lowerCamelCase   :							Tuple=0.1				,			_lowerCamelCase   :							Dict=512				,			_lowerCamelCase   :							List[Any]=0.02				,			):





											"""simple docstring"""







											A_			:  Tuple			=				parent
											A_			:  Dict			=				batch_size
											A_			:  List[str]			=				seq_length
											A_			:  Optional[Any]			=				is_training
											A_			:  str			=				use_input_mask
											A_			:  List[str]			=				use_token_type_ids
											A_			:  int			=				use_labels
											A_			:  Any			=				vocab_size
											A_			:  List[str]			=				hidden_size
											A_			:  Union[str, Any]			=				num_hidden_layers
											A_			:  str			=				num_attention_heads
											A_			:  int			=				intermediate_size
											A_			:  Tuple			=				hidden_act
											A_			:  Optional[Any]			=				hidden_dropout_prob
											A_			:  List[Any]			=				attention_probs_dropout_prob
											A_			:  Dict			=				max_position_embeddings
											A_			:  Union[str, Any]			=				initializer_range
											A_			:  List[str]			=				None
											A_			:  str			=				vocab_size - 1
											A_			:  Tuple			=				vocab_size - 1
											A_			:  Optional[Any]			=				vocab_size - 1







						def    _a     (  self   :							Any     ):





											"""simple docstring"""







											A_			:  Dict			=				ids_tensor([self.batch_size, self.seq_length]				,			self.vocab_size     )

											A_			:  Tuple			=				None
											if self.use_input_mask:
																A_			:  int			=				random_attention_mask([self.batch_size, self.seq_length]     )

											A_			:  Optional[int]			=				GPTaConfig(
											    vocab_size=self.vocab_size				,			n_embd=self.hidden_size				,			n_layer=self.num_hidden_layers				,			n_head=self.num_attention_heads				,			n_positions=self.max_position_embeddings				,			use_cache=_lowerCamelCase				,			bos_token_id=self.bos_token_id				,			eos_token_id=self.eos_token_id				,			pad_token_id=self.pad_token_id				,			)

											return (config, input_ids, input_mask)







						def    _a     (  self   :							int     ):





											"""simple docstring"""







											A_			:  Union[str, Any]			=				self.prepare_config_and_inputs()
											A_			,A_			,A_			:  Optional[Any]			=				config_and_inputs
											A_			:  Union[str, Any]			=				{'''input_ids''': input_ids, '''attention_mask''': attention_mask}
											return config, inputs_dict







						def    _a     (  self   :							List[str]     ):





											"""simple docstring"""







											A_			,A_			,A_			:  Optional[int]			=				self.prepare_config_and_inputs()

											A_			:  str			=				floats_tensor([self.batch_size, self.seq_length, self.hidden_size]     )
											A_			:  List[Any]			=				ids_tensor([self.batch_size, self.seq_length]				,			vocab_size=2     )

											return (
											    config,
											    input_ids,
											    attention_mask,
											    encoder_hidden_states,
											    encoder_attention_mask,
											)







						def    _a     (  self   :							List[Any]				,			_lowerCamelCase   :							str				,			_lowerCamelCase   :							str				,			_lowerCamelCase   :							str				,			_lowerCamelCase   :							Dict     ):





											"""simple docstring"""







											A_			:  Optional[Any]			=				20
											A_			:  Dict			=				model_class_name(_lowerCamelCase     )

											A_			:  Tuple			=				model.init_cache(input_ids.shape[0]				,			_lowerCamelCase     )
											A_			:  Any			=				jnp.ones((input_ids.shape[0], max_decoder_length)				,			dtype='''i4'''     )

											A_			:  Dict			=				jnp.broadcast_to(
											    jnp.arange(input_ids.shape[-1] - 1     )[None, :]				,			(input_ids.shape[0], input_ids.shape[-1] - 1)     )
											A_			:  List[Any]			=				model(
											    input_ids[:, :-1]				,			attention_mask=_lowerCamelCase				,			past_key_values=_lowerCamelCase				,			position_ids=_lowerCamelCase				,			)

											A_			:  Optional[int]			=				jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]				,			dtype='''i4'''     )
											A_			:  Any			=				model(
											    input_ids[:, -1:]				,			attention_mask=_lowerCamelCase				,			past_key_values=outputs_cache.past_key_values				,			position_ids=_lowerCamelCase				,			)

											A_			:  Any			=				model(_lowerCamelCase     )

											A_			:  str			=				np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])     )     )
											self.parent.assertTrue(diff < 1E-3				,			msg=f'Max diff is {diff}'     )







						def    _a     (  self   :							List[str]				,			_lowerCamelCase   :							Dict				,			_lowerCamelCase   :							int				,			_lowerCamelCase   :							Optional[Any]				,			_lowerCamelCase   :							Optional[Any]     ):





											"""simple docstring"""







											A_			:  Any			=				20
											A_			:  Dict			=				model_class_name(_lowerCamelCase     )

											A_			:  str			=				jnp.concatenate(
											    [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1])     )]				,			axis=-1				,			)

											A_			:  Dict			=				model.init_cache(input_ids.shape[0]				,			_lowerCamelCase     )
											A_			:  List[str]			=				jnp.broadcast_to(
											    jnp.arange(input_ids.shape[-1] - 1     )[None, :]				,			(input_ids.shape[0], input_ids.shape[-1] - 1)     )

											A_			:  Optional[int]			=				model(
											    input_ids[:, :-1]				,			attention_mask=_lowerCamelCase				,			past_key_values=_lowerCamelCase				,			position_ids=_lowerCamelCase				,			)
											A_			:  Optional[int]			=				jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]				,			dtype='''i4'''     )
											A_			:  int			=				model(
											    input_ids[:, -1:]				,			past_key_values=outputs_cache.past_key_values				,			attention_mask=_lowerCamelCase				,			position_ids=_lowerCamelCase				,			)

											A_			:  Dict			=				model(_lowerCamelCase				,			attention_mask=_lowerCamelCase     )

											A_			:  Union[str, Any]			=				np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])     )     )
											self.parent.assertTrue(diff < 1E-3				,			msg=f'Max diff is {diff}'     )






@require_flax
class      UpperCamelCase_  (a__,   a__,   unittest.TestCase  ):







						"""simple docstring"""


						_lowerCAmelCase         =       (FlaxGPTaModel, FlaxGPTaLMHeadModel) if is_flax_available() else ()
						_lowerCAmelCase         =       (FlaxGPTaLMHeadModel,) if is_flax_available() else ()







						def    _a     (  self   :							Tuple     ):





											"""simple docstring"""







											A_			:  List[str]			=				FlaxGPTaModelTester(self     )







						def    _a     (  self   :							str     ):





											"""simple docstring"""







											for model_class_name in self.all_model_classes:
																A_			,A_			,A_			:  Any			=				self.model_tester.prepare_config_and_inputs()
																self.model_tester.check_use_cache_forward(_lowerCamelCase				,			_lowerCamelCase				,			_lowerCamelCase				,			_lowerCamelCase     )







						def    _a     (  self   :							int     ):





											"""simple docstring"""







											for model_class_name in self.all_model_classes:
																A_			,A_			,A_			:  List[str]			=				self.model_tester.prepare_config_and_inputs()
																self.model_tester.check_use_cache_forward_with_attn_mask(
																    _lowerCamelCase				,			_lowerCamelCase				,			_lowerCamelCase				,			_lowerCamelCase     )







						@slow
						def    _a     (  self   :							Any     ):





											"""simple docstring"""







											A_			:  Optional[Any]			=				GPTaTokenizer.from_pretrained('''gpt2'''				,			pad_token='''</s>'''				,			padding_side='''left'''     )
											A_			:  Tuple			=				tokenizer(['''Hello this is a long string''', '''Hey''']				,			return_tensors='''np'''				,			padding=_lowerCamelCase				,			truncation=_lowerCamelCase     )

											A_			:  Any			=				FlaxGPTaLMHeadModel.from_pretrained('''gpt2'''     )
											A_			:  int			=				False
											A_			:  Union[str, Any]			=				model.config.eos_token_id

											A_			:  List[Any]			=				jax.jit(model.generate     )

											A_			:  List[Any]			=				jit_generate(inputs['''input_ids''']				,			attention_mask=inputs['''attention_mask''']     ).sequences

											A_			:  List[Any]			=				tokenizer.batch_decode(_lowerCamelCase				,			skip_special_tokens=_lowerCamelCase     )

											A_			:  Optional[Any]			=				[
											    '''Hello this is a long string of words. I\'m going to start with the first one.\n''',
											    '''Hey, I\'m not sure if I\'m going to be able to do''',
											]

											self.assertListEqual(_lowerCamelCase				,			_lowerCamelCase     )







						@is_pt_flax_cross_test
						def    _a     (  self   :							Optional[Any]     ):





											"""simple docstring"""







											A_			,A_			:  Tuple			=				self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
																with self.subTest(model_class.__name__     ):
																					# prepare inputs
																					A_			:  Optional[Any]			=				self._prepare_for_class(_lowerCamelCase				,			_lowerCamelCase     )
																					A_			:  List[str]			=				{k: torch.tensor(v.tolist()     ) for k, v in prepared_inputs_dict.items()}

																					# load corresponding PyTorch class
																					A_			:  List[str]			=				model_class.__name__[4:]  # Skip the "Flax" at the beginning
																					A_			:  int			=				getattr(_lowerCamelCase				,			_lowerCamelCase     )

																					A_			,A_			:  int			=				pt_inputs['''input_ids'''].shape
																					A_			:  Dict			=				np.random.randint(0				,			seq_length - 1				,			size=(batch_size,)     )
																					for batch_idx, start_index in enumerate(_lowerCamelCase     ):
																										A_			:  Optional[int]			=				0
																										A_			:  List[str]			=				1
																										A_			:  Optional[int]			=				0
																										A_			:  Union[str, Any]			=				1
																					A_			:  List[str]			=				pt_model_class(_lowerCamelCase     ).eval()
																					A_			:  List[str]			=				model_class(_lowerCamelCase				,			dtype=jnp.floataa     )

																					A_			:  List[str]			=				convert_pytorch_state_dict_to_flax(pt_model.state_dict()				,			_lowerCamelCase     )
																					A_			:  Any			=				fx_state

																					with torch.no_grad():
																										A_			:  Optional[int]			=				pt_model(**_lowerCamelCase     ).to_tuple()

																					A_			:  Any			=				fx_model(**_lowerCamelCase     ).to_tuple()
																					self.assertEqual(len(_lowerCamelCase     )				,			len(_lowerCamelCase     )				,			'''Output lengths differ between Flax and PyTorch'''     )
																					for fx_output, pt_output in zip(_lowerCamelCase				,			_lowerCamelCase     ):
																										self.assert_almost_equals(fx_output[:, -1]				,			pt_output[:, -1].numpy()				,			4E-2     )

																					with tempfile.TemporaryDirectory() as tmpdirname:
																										pt_model.save_pretrained(_lowerCamelCase     )
																										A_			:  Optional[int]			=				model_class.from_pretrained(_lowerCamelCase				,			from_pt=_lowerCamelCase     )

																					A_			:  Optional[int]			=				fx_model_loaded(**_lowerCamelCase     ).to_tuple()
																					self.assertEqual(
																					    len(_lowerCamelCase     )				,			len(_lowerCamelCase     )				,			'''Output lengths differ between Flax and PyTorch'''     )
																					for fx_output_loaded, pt_output in zip(_lowerCamelCase				,			_lowerCamelCase     ):
																										self.assert_almost_equals(fx_output_loaded[:, -1]				,			pt_output[:, -1].numpy()				,			4E-2     )







						@is_pt_flax_cross_test
						def    _a     (  self   :							Tuple     ):





											"""simple docstring"""







											A_			,A_			:  Optional[Any]			=				self.model_tester.prepare_config_and_inputs_for_common()
											for model_class in self.all_model_classes:
																with self.subTest(model_class.__name__     ):
																					# prepare inputs
																					A_			:  Optional[Any]			=				self._prepare_for_class(_lowerCamelCase				,			_lowerCamelCase     )
																					A_			:  Any			=				{k: torch.tensor(v.tolist()     ) for k, v in prepared_inputs_dict.items()}

																					# load corresponding PyTorch class
																					A_			:  Any			=				model_class.__name__[4:]  # Skip the "Flax" at the beginning
																					A_			:  Union[str, Any]			=				getattr(_lowerCamelCase				,			_lowerCamelCase     )

																					A_			:  Any			=				pt_model_class(_lowerCamelCase     ).eval()
																					A_			:  Dict			=				model_class(_lowerCamelCase				,			dtype=jnp.floataa     )

																					A_			:  Union[str, Any]			=				load_flax_weights_in_pytorch_model(_lowerCamelCase				,			fx_model.params     )
																					A_			,A_			:  Tuple			=				pt_inputs['''input_ids'''].shape
																					A_			:  Dict			=				np.random.randint(0				,			seq_length - 1				,			size=(batch_size,)     )
																					for batch_idx, start_index in enumerate(_lowerCamelCase     ):
																										A_			:  Optional[Any]			=				0
																										A_			:  Union[str, Any]			=				1
																										A_			:  Any			=				0
																										A_			:  Tuple			=				1

																					# make sure weights are tied in PyTorch
																					pt_model.tie_weights()

																					with torch.no_grad():
																										A_			:  Optional[Any]			=				pt_model(**_lowerCamelCase     ).to_tuple()

																					A_			:  str			=				fx_model(**_lowerCamelCase     ).to_tuple()
																					self.assertEqual(len(_lowerCamelCase     )				,			len(_lowerCamelCase     )				,			'''Output lengths differ between Flax and PyTorch'''     )
																					for fx_output, pt_output in zip(_lowerCamelCase				,			_lowerCamelCase     ):
																										self.assert_almost_equals(fx_output[:, -1]				,			pt_output[:, -1].numpy()				,			4E-2     )

																					with tempfile.TemporaryDirectory() as tmpdirname:
																										fx_model.save_pretrained(_lowerCamelCase     )
																										A_			:  int			=				pt_model_class.from_pretrained(_lowerCamelCase				,			from_flax=_lowerCamelCase     )

																					with torch.no_grad():
																										A_			:  Dict			=				pt_model_loaded(**_lowerCamelCase     ).to_tuple()

																					self.assertEqual(
																					    len(_lowerCamelCase     )				,			len(_lowerCamelCase     )				,			'''Output lengths differ between Flax and PyTorch'''     )
																					for fx_output, pt_output in zip(_lowerCamelCase				,			_lowerCamelCase     ):
																										self.assert_almost_equals(fx_output[:, -1]				,			pt_output[:, -1].numpy()				,			4E-2     )







						@slow
						def    _a     (  self   :							Optional[Any]     ):





											"""simple docstring"""







											for model_class_name in self.all_model_classes:
																A_			:  Tuple			=				model_class_name.from_pretrained('''gpt2'''				,			from_pt=_lowerCamelCase     )
																A_			:  Dict			=				model(np.ones((1, 1)     )     )
																self.assertIsNotNone(_lowerCamelCase     )









'''simple docstring'''




import jax
import jax.numpy as jnp
from bigbird_flax import FlaxBigBirdForNaturalQuestions
from datasets import load_from_disk

from transformers import BigBirdTokenizerFast


snake_case__ 							=    {0: """null""", 1: """short""", 2: """long""", 3: """yes""", 4: """no"""}
snake_case__ 							=    set("""""".join(["""‘""", """’""", """´""", """`""", """.""", """,""", """-""", """\""""]))




def     snake_case__     (			lowerCamelCase__					: List[str]      ,		lowerCamelCase__					: Tuple=0      ,		lowerCamelCase__					: Tuple=None      ) -> Optional[Any]:
					return [" ".join(x.split(''' '''      )[begin:end]      ) for x in answers if len(x.split(''' '''      )      ) > 1]




def     snake_case__     (			lowerCamelCase__					: Any      ,		lowerCamelCase__					: Any=False      ) -> List[Any]:
					if make_sub_answers:
										# if answers are longer than one word, make sure a predictions is correct if it coresponds to the complete 1: or :-1 sub word
										# *e.g.* if the correct answer contains a prefix such as "the", or "a"
										A_			:  Union[str, Any]			=				(
										    given_answers + get_sub_answers(lowerCamelCase__      ,		begin=1      ) + get_sub_answers(lowerCamelCase__      ,		end=-1      )
										)
					A_			:  List[Any]			=				[]
					for answer in given_answers:
										A_			:  Dict			=				answer.replace('''_'''      ,		''' '''      ).lower()
										A_			:  str			=				''''''.join(c if c not in PUNCTUATION_SET_TO_EXCLUDE else ''' ''' for c in alias      )
										answers.append(''' '''.join(alias.split()      ).strip()      )
					return set(lowerCamelCase__      )




def     snake_case__     (			lowerCamelCase__					: Optional[Any]      ,		lowerCamelCase__					: Dict      ,		lowerCamelCase__					: Union[str, Any]=1      ,		lowerCamelCase__					: Any=1_0_0      ) -> Any:
					A_			,A_			:  str			=				jax.lax.top_k(lowerCamelCase__      ,		lowerCamelCase__      )
					A_			,A_			:  str			=				jax.lax.top_k(lowerCamelCase__      ,		lowerCamelCase__      )

					A_			:  List[Any]			=				best_end_idx[:, None] - best_start_idx[None, :]
					A_			:  Optional[Any]			=				jnp.logical_or(widths < 0      ,		widths > max_size      )
					A_			:  Optional[int]			=				(best_end_scores[:, None] + best_start_scores[None, :]) - (1e8 * mask)
					A_			:  str			=				jnp.argmax(lowerCamelCase__      ).item()

					return best_start_idx[best_score % top_k], best_end_idx[best_score // top_k]




def     snake_case__     (			lowerCamelCase__					: List[str]      ) -> Union[str, Any]:
					A_			:  str			=				sample['''question''']['''text''']
					A_			:  Tuple			=				sample['''document''']['''tokens''']['''token''']
					A_			:  List[Any]			=				sample['''document''']['''tokens''']['''is_html''']
					A_			:  Tuple			=				sample['''annotations''']['''long_answer''']
					A_			:  List[Any]			=				sample['''annotations''']['''short_answers''']

					A_			:  Optional[Any]			=				''' '''.join([context[i] for i in range(len(lowerCamelCase__      )      ) if not is_html[i]]      )

					# 0 - No ; 1 - Yes
					for answer in sample["annotations"]["yes_no_answer"]:
										if answer == 0 or answer == 1:
															return {
															    "question": question,
															    "context": context_string,
															    "short": [],
															    "long": [],
															    "category": "no" if answer == 0 else "yes",
															}

					A_			:  List[Any]			=				[]
					for s in short_answers:
										short_targets.extend(s['''text''']      )
					A_			:  Tuple			=				list(set(lowerCamelCase__      )      )

					A_			:  Tuple			=				[]
					for s in long_answers:
										if s["start_token"] == -1:
															continue
										A_			:  Dict			=				context[s['''start_token'''] : s['''end_token''']]
										A_			:  Tuple			=				is_html[s['''start_token'''] : s['''end_token''']]
										A_			:  Optional[Any]			=				''' '''.join([answer[i] for i in range(len(lowerCamelCase__      )      ) if not html[i]]      )
										if new_answer not in long_targets:
															long_targets.append(lowerCamelCase__      )

					A_			:  str			=				'''long_short''' if len(short_targets + long_targets      ) > 0 else '''null'''

					return {
					    "question": question,
					    "context": context_string,
					    "short": short_targets,
					    "long": long_targets,
					    "category": category,
					}




def     snake_case__     (			) -> List[Any]:
					A_			:  int			=				load_from_disk('''natural-questions-validation'''      )
					A_			:  Optional[int]			=				dataset.map(lowerCamelCase__      ).remove_columns(['''annotations''', '''document''', '''id''']      )
					print(lowerCamelCase__      )

					A_			:  Tuple			=				dataset.filter(lambda lowerCamelCase__      : (len(x['''question''']      ) + len(x['''context''']      )) < 4 * 4_0_9_6      )
					A_			:  Optional[Any]			=				short_validation_dataset.filter(lambda lowerCamelCase__      : x["category"] != "null"      )
					short_validation_dataset

					A_			:  Union[str, Any]			=				'''vasudevgupta/flax-bigbird-natural-questions'''
					A_			:  List[str]			=				FlaxBigBirdForNaturalQuestions.from_pretrained(lowerCamelCase__      )
					A_			:  int			=				BigBirdTokenizerFast.from_pretrained(lowerCamelCase__      )

					@jax.jit
					def forward(*lowerCamelCase__					: Any      ,		**lowerCamelCase__					: Optional[Any]      ):
										A_			,A_			,A_			:  Tuple			=				model(*lowerCamelCase__      ,		**lowerCamelCase__      )
										return start_logits, end_logits, jnp.argmax(lowerCamelCase__      ,		axis=-1      )

					def evaluate(lowerCamelCase__					: List[str]      ):
										# encode question and context so that they are separated by a tokenizer.sep_token and cut at max_length
										A_			:  List[Any]			=				tokenizer(
										    example['''question''']      ,		example['''context''']      ,		return_tensors='''np'''      ,		max_length=4_0_9_6      ,		padding='''max_length'''      ,		truncation=lowerCamelCase__      ,		)

										A_			,A_			,A_			:  Optional[Any]			=				forward(**lowerCamelCase__      )

										A_			:  Optional[Any]			=				CATEGORY_MAPPING[category.item()]

										A_			:  str			=				example['''long'''] + example['''short''']
										if example["category"] in ["yes", "no", "null"]:
															A_			:  str			=				[example['''category''']]
										A_			:  Optional[Any]			=				example['''category'''] != '''null'''
										# Now target can be: "yes", "no", "null", "list of long & short answers"

										if predicted_category in ["yes", "no", "null"]:
															A_			:  int			=				[predicted_category]
															A_			:  Optional[int]			=				example['''output'''] == example['''targets''']
															A_			:  Dict			=				predicted_category != '''null'''
															return example

										A_			:  List[Any]			=				3_8 if predicted_category == '''short''' else 1_0_2_4
										A_			,A_			:  str			=				get_best_valid_start_end_idx(
										    start_scores[0]      ,		end_scores[0]      ,		top_k=8      ,		max_size=lowerCamelCase__      )

										A_			:  List[Any]			=				inputs['''input_ids'''][0].tolist()
										A_			:  Dict			=				[tokenizer.decode(input_ids[start_score : end_score + 1]      )]

										A_			:  Optional[int]			=				expand_to_aliases(example['''targets''']      ,		make_sub_answers=lowerCamelCase__      )
										A_			:  str			=				expand_to_aliases(example['''output''']      )

										# some preprocessing to both prediction and answer
										A_			:  Tuple			=				{''''''.join(a.split()      ) for a in answers}
										A_			:  Optional[int]			=				{''''''.join(p.split()      ) for p in predictions}
										A_			:  List[str]			=				{s for s in predictions if s not in ['''``''', '''\'\'''', '''`''', '''\'''']}

										# if there is a common element, it's a exact match
										A_			:  List[Any]			=				len(list(answers & predictions      )      ) > 0
										A_			:  Dict			=				predicted_category != '''null''' and len(lowerCamelCase__      ) > 0

										return example

					A_			:  Dict			=				short_validation_dataset.map(lowerCamelCase__      )

					A_			:  List[str]			=				len(lowerCamelCase__      )
					A_			:  Any			=				len(short_validation_dataset.filter(lambda lowerCamelCase__      : x["match"] == 1      )      )
					print('''EM score:'''      ,		(matched / total) * 1_0_0      ,		'''%'''      )


if __name__ == "__main__":
							main()



