import unittest

from transformers import ElectraConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          MODEL_FOR_PRETRAINING_MAPPING,
          ElectraForCausalLM,
          ElectraForMaskedLM,
          ElectraForMultipleChoice,
          ElectraForPreTraining,
          ElectraForQuestionAnswering,
          ElectraForSequenceClassification,
          ElectraForTokenClassification,
          ElectraModel,
      )
      from transformers.models.electra.modeling_electra import ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST



class     snake_case_			:






   def __init__( self							:	Union[str, Any]      ,		lowercase_							:	Optional[int]      ,		lowercase_							:	Any=13      ,		lowercase_							:	Optional[int]=7      ,		lowercase_							:	int=True      ,		lowercase_							:	Dict=True      ,		lowercase_							:	int=True      ,		lowercase_							:	List[str]=True      ,		lowercase_							:	List[Any]=99      ,		lowercase_							:	List[str]=32      ,		lowercase_							:	Optional[Any]=5      ,		lowercase_							:	List[str]=4      ,		lowercase_							:	str=37      ,		lowercase_							:	Union[str, Any]="gelu"      ,		lowercase_							:	Any=0.1      ,		lowercase_							:	Optional[int]=0.1      ,		lowercase_							:	Tuple=5_12      ,		lowercase_							:	Tuple=16      ,		lowercase_							:	Optional[int]=2      ,		lowercase_							:	Optional[int]=0.02      ,		lowercase_							:	List[Any]=3      ,		lowercase_							:	Optional[int]=4      ,		lowercase_							:	Optional[Any]=None      ,		)   ->		Optional[Any]:
       lowercase__					:  Optional[int]          =							parent
       lowercase__					:  List[Any]          =							batch_size
       lowercase__					:  Optional[int]          =							seq_length
       lowercase__					:  Dict          =							is_training
       lowercase__					:  Union[str, Any]          =							use_input_mask
       lowercase__					:  Tuple          =							use_token_type_ids
       lowercase__					:  str          =							use_labels
       lowercase__					:  List[str]          =							vocab_size
       lowercase__					:  Tuple          =							hidden_size
       lowercase__					:  Tuple          =							num_hidden_layers
       lowercase__					:  List[str]          =							num_attention_heads
       lowercase__					:  Optional[Any]          =							intermediate_size
       lowercase__					:  str          =							hidden_act
       lowercase__					:  Optional[int]          =							hidden_dropout_prob
       lowercase__					:  int          =							attention_probs_dropout_prob
       lowercase__					:  Optional[Any]          =							max_position_embeddings
       lowercase__					:  List[Any]          =							type_vocab_size
       lowercase__					:  Tuple          =							type_sequence_label_size
       lowercase__					:  Any          =							initializer_range
       lowercase__					:  Any          =							num_labels
       lowercase__					:  str          =							num_choices
       lowercase__					:  List[str]          =							scope






   def  __UpperCamelCase	( self							:	int    )   ->		Tuple:
       lowercase__					:  List[Any]          =							ids_tensor([self.batch_size, self.seq_length]      ,		self.vocab_size    )

       lowercase__					:  Union[str, Any]          =							None
       if self.use_input_mask:
           lowercase__					:  Tuple          =							random_attention_mask([self.batch_size, self.seq_length]    )

       lowercase__					:  List[str]          =							None
       if self.use_token_type_ids:
           lowercase__					:  List[Any]          =							ids_tensor([self.batch_size, self.seq_length]      ,		self.type_vocab_size    )

       lowercase__					:  Tuple          =							None
       lowercase__					:  Dict          =							None
       lowercase__					:  int          =							None
       if self.use_labels:
           lowercase__					:  Any          =							ids_tensor([self.batch_size]      ,		self.type_sequence_label_size    )
           lowercase__					:  Dict          =							ids_tensor([self.batch_size, self.seq_length]      ,		self.num_labels    )
           lowercase__					:  int          =							ids_tensor([self.batch_size]      ,		self.num_choices    )
           lowercase__					:  List[str]          =							ids_tensor([self.batch_size, self.seq_length]      ,		1    )

       lowercase__					:  List[Any]          =							self.get_config()

       return (
           config,
           input_ids,
           token_type_ids,
           input_mask,
           sequence_labels,
           token_labels,
           choice_labels,
           fake_token_labels,
       )






   def  __UpperCamelCase	( self							:	Optional[Any]    )   ->		Tuple:
       return ElectraConfig(
           vocab_size=self.vocab_size      ,		hidden_size=self.hidden_size      ,		num_hidden_layers=self.num_hidden_layers      ,		num_attention_heads=self.num_attention_heads      ,		intermediate_size=self.intermediate_size      ,		hidden_act=self.hidden_act      ,		hidden_dropout_prob=self.hidden_dropout_prob      ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,		max_position_embeddings=self.max_position_embeddings      ,		type_vocab_size=self.type_vocab_size      ,		is_decoder=UpperCAmelCase__      ,		initializer_range=self.initializer_range      ,		)






   def  __UpperCamelCase	( self							:	Dict    )   ->		int:
       (
           lowercase__       
       )					:  List[str]          =							self.prepare_config_and_inputs()

       lowercase__					:  Tuple          =							True
       lowercase__					:  List[str]          =							floats_tensor([self.batch_size, self.seq_length, self.hidden_size]    )
       lowercase__					:  str          =							ids_tensor([self.batch_size, self.seq_length]      ,		vocab_size=2    )

       return (
           config,
           input_ids,
           token_type_ids,
           input_mask,
           sequence_labels,
           token_labels,
           choice_labels,
           encoder_hidden_states,
           encoder_attention_mask,
       )






   def  __UpperCamelCase	( self							:	Optional[Any]      ,		lowercase_							:	List[Any]      ,		lowercase_							:	Tuple      ,		lowercase_							:	List[str]      ,		lowercase_							:	Union[str, Any]      ,		lowercase_							:	Optional[int]      ,		lowercase_							:	Optional[Any]      ,		lowercase_							:	Optional[Any]      ,		lowercase_							:	Optional[int]      ,		)   ->		Tuple:
       lowercase__					:  List[str]          =							ElectraModel(config=UpperCAmelCase__    )
       model.to(UpperCAmelCase__    )
       model.eval()
       lowercase__					:  Tuple          =							model(UpperCAmelCase__      ,		attention_mask=UpperCAmelCase__      ,		token_type_ids=UpperCAmelCase__    )
       lowercase__					:  List[Any]          =							model(UpperCAmelCase__      ,		token_type_ids=UpperCAmelCase__    )
       lowercase__					:  Tuple          =							model(UpperCAmelCase__    )
       self.parent.assertEqual(result.last_hidden_state.shape      ,		(self.batch_size, self.seq_length, self.hidden_size)    )






   def  __UpperCamelCase	( self							:	Optional[Any]      ,		lowercase_							:	Optional[int]      ,		lowercase_							:	Any      ,		lowercase_							:	Any      ,		lowercase_							:	int      ,		lowercase_							:	List[str]      ,		lowercase_							:	Optional[int]      ,		lowercase_							:	str      ,		lowercase_							:	Optional[Any]      ,		lowercase_							:	Dict      ,		)   ->		List[str]:
       lowercase__					:  Dict          =							True
       lowercase__					:  Any          =							ElectraModel(UpperCAmelCase__    )
       model.to(UpperCAmelCase__    )
       model.eval()
       lowercase__					:  List[str]          =							model(
           UpperCAmelCase__      ,		attention_mask=UpperCAmelCase__      ,		token_type_ids=UpperCAmelCase__      ,		encoder_hidden_states=UpperCAmelCase__      ,		encoder_attention_mask=UpperCAmelCase__      ,		)
       lowercase__					:  Optional[int]          =							model(
           UpperCAmelCase__      ,		attention_mask=UpperCAmelCase__      ,		token_type_ids=UpperCAmelCase__      ,		encoder_hidden_states=UpperCAmelCase__      ,		)
       lowercase__					:  List[Any]          =							model(UpperCAmelCase__      ,		attention_mask=UpperCAmelCase__      ,		token_type_ids=UpperCAmelCase__    )
       self.parent.assertEqual(result.last_hidden_state.shape      ,		(self.batch_size, self.seq_length, self.hidden_size)    )






   def  __UpperCamelCase	( self							:	str      ,		lowercase_							:	List[str]      ,		lowercase_							:	Optional[Any]      ,		lowercase_							:	Union[str, Any]      ,		lowercase_							:	Dict      ,		lowercase_							:	Optional[int]      ,		lowercase_							:	int      ,		lowercase_							:	Optional[int]      ,		lowercase_							:	List[str]      ,		)   ->		List[str]:
       lowercase__					:  Optional[int]          =							ElectraForMaskedLM(config=UpperCAmelCase__    )
       model.to(UpperCAmelCase__    )
       model.eval()
       lowercase__					:  List[str]          =							model(UpperCAmelCase__      ,		attention_mask=UpperCAmelCase__      ,		token_type_ids=UpperCAmelCase__      ,		labels=UpperCAmelCase__    )
       self.parent.assertEqual(result.logits.shape      ,		(self.batch_size, self.seq_length, self.vocab_size)    )






   def  __UpperCamelCase	( self							:	List[Any]      ,		lowercase_							:	Tuple      ,		lowercase_							:	Optional[Any]      ,		lowercase_							:	Union[str, Any]      ,		lowercase_							:	Optional[int]      ,		lowercase_							:	Tuple      ,		lowercase_							:	List[str]      ,		lowercase_							:	Dict      ,		lowercase_							:	List[Any]      ,		lowercase_							:	List[str]      ,		)   ->		Union[str, Any]:
       lowercase__					:  List[Any]          =							ElectraForCausalLM(config=UpperCAmelCase__    )
       model.to(UpperCAmelCase__    )
       model.eval()
       lowercase__					:  Optional[int]          =							model(UpperCAmelCase__      ,		attention_mask=UpperCAmelCase__      ,		token_type_ids=UpperCAmelCase__      ,		labels=UpperCAmelCase__    )
       self.parent.assertEqual(result.logits.shape      ,		(self.batch_size, self.seq_length, self.vocab_size)    )






   def  __UpperCamelCase	( self							:	Any      ,		lowercase_							:	Dict      ,		lowercase_							:	int      ,		lowercase_							:	str      ,		lowercase_							:	str      ,		lowercase_							:	Union[str, Any]      ,		lowercase_							:	Tuple      ,		lowercase_							:	List[str]      ,		lowercase_							:	str      ,		)   ->		List[str]:
       lowercase__					:  str          =							self.num_labels
       lowercase__					:  str          =							ElectraForTokenClassification(config=UpperCAmelCase__    )
       model.to(UpperCAmelCase__    )
       model.eval()
       lowercase__					:  str          =							model(UpperCAmelCase__      ,		attention_mask=UpperCAmelCase__      ,		token_type_ids=UpperCAmelCase__      ,		labels=UpperCAmelCase__    )
       self.parent.assertEqual(result.logits.shape      ,		(self.batch_size, self.seq_length, self.num_labels)    )






   def  __UpperCamelCase	( self							:	str      ,		lowercase_							:	Optional[int]      ,		lowercase_							:	Any      ,		lowercase_							:	List[str]      ,		lowercase_							:	List[Any]      ,		lowercase_							:	int      ,		lowercase_							:	Optional[Any]      ,		lowercase_							:	Dict      ,		lowercase_							:	List[Any]      ,		)   ->		List[Any]:
       lowercase__					:  Optional[int]          =							self.num_labels
       lowercase__					:  Optional[int]          =							ElectraForPreTraining(config=UpperCAmelCase__    )
       model.to(UpperCAmelCase__    )
       model.eval()
       lowercase__					:  Dict          =							model(UpperCAmelCase__      ,		attention_mask=UpperCAmelCase__      ,		token_type_ids=UpperCAmelCase__      ,		labels=UpperCAmelCase__    )
       self.parent.assertEqual(result.logits.shape      ,		(self.batch_size, self.seq_length)    )






   def  __UpperCamelCase	( self							:	str      ,		lowercase_							:	List[Any]      ,		lowercase_							:	List[str]      ,		lowercase_							:	Optional[int]      ,		lowercase_							:	List[Any]      ,		lowercase_							:	List[Any]      ,		lowercase_							:	Any      ,		lowercase_							:	Optional[int]      ,		lowercase_							:	Union[str, Any]      ,		)   ->		Tuple:
       lowercase__					:  Optional[Any]          =							self.num_labels
       lowercase__					:  Optional[Any]          =							ElectraForSequenceClassification(UpperCAmelCase__    )
       model.to(UpperCAmelCase__    )
       model.eval()
       lowercase__					:  Any          =							model(UpperCAmelCase__      ,		attention_mask=UpperCAmelCase__      ,		token_type_ids=UpperCAmelCase__      ,		labels=UpperCAmelCase__    )
       self.parent.assertEqual(result.logits.shape      ,		(self.batch_size, self.num_labels)    )






   def  __UpperCamelCase	( self							:	Optional[Any]      ,		lowercase_							:	Dict      ,		lowercase_							:	List[Any]      ,		lowercase_							:	Optional[Any]      ,		lowercase_							:	List[Any]      ,		lowercase_							:	List[str]      ,		lowercase_							:	Any      ,		lowercase_							:	Union[str, Any]      ,		lowercase_							:	int      ,		)   ->		List[Any]:
       lowercase__					:  Optional[int]          =							ElectraForQuestionAnswering(config=UpperCAmelCase__    )
       model.to(UpperCAmelCase__    )
       model.eval()
       lowercase__					:  Union[str, Any]          =							model(
           UpperCAmelCase__      ,		attention_mask=UpperCAmelCase__      ,		token_type_ids=UpperCAmelCase__      ,		start_positions=UpperCAmelCase__      ,		end_positions=UpperCAmelCase__      ,		)
       self.parent.assertEqual(result.start_logits.shape      ,		(self.batch_size, self.seq_length)    )
       self.parent.assertEqual(result.end_logits.shape      ,		(self.batch_size, self.seq_length)    )






   def  __UpperCamelCase	( self							:	str      ,		lowercase_							:	Tuple      ,		lowercase_							:	Tuple      ,		lowercase_							:	str      ,		lowercase_							:	Optional[Any]      ,		lowercase_							:	Optional[Any]      ,		lowercase_							:	List[str]      ,		lowercase_							:	Any      ,		lowercase_							:	Optional[Any]      ,		)   ->		Union[str, Any]:
       lowercase__					:  Union[str, Any]          =							self.num_choices
       lowercase__					:  Union[str, Any]          =							ElectraForMultipleChoice(config=UpperCAmelCase__    )
       model.to(UpperCAmelCase__    )
       model.eval()
       lowercase__					:  Dict          =							input_ids.unsqueeze(1    ).expand(-1      ,		self.num_choices      ,		-1    ).contiguous()
       lowercase__					:  str          =							token_type_ids.unsqueeze(1    ).expand(-1      ,		self.num_choices      ,		-1    ).contiguous()
       lowercase__					:  Dict          =							input_mask.unsqueeze(1    ).expand(-1      ,		self.num_choices      ,		-1    ).contiguous()
       lowercase__					:  str          =							model(
           UpperCAmelCase__      ,		attention_mask=UpperCAmelCase__      ,		token_type_ids=UpperCAmelCase__      ,		labels=UpperCAmelCase__      ,		)
       self.parent.assertEqual(result.logits.shape      ,		(self.batch_size, self.num_choices)    )







   def  __UpperCamelCase	( self							:	Any    )   ->		List[str]:
       lowercase__					:  Optional[Any]          =							self.prepare_config_and_inputs()
       (
           lowercase__       
       )					:  int          =							config_and_inputs
       lowercase__					:  Dict          =							{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
       return config, inputs_dict



@require_torch
class     snake_case_			(						snake_case_    ,snake_case_    ,unittest.TestCase			):
   __A      :						Union[str, Any]   =							(
       (
           ElectraModel,
           ElectraForPreTraining,
           ElectraForMaskedLM,
           ElectraForCausalLM,
           ElectraForMultipleChoice,
           ElectraForTokenClassification,
           ElectraForSequenceClassification,
           ElectraForQuestionAnswering,
       )
       if is_torch_available()
       else ()
   )
   __A      :						Any   =							(
       {
           "feature-extraction": ElectraModel,
           "fill-mask": ElectraForMaskedLM,
           "question-answering": ElectraForQuestionAnswering,
           "text-classification": ElectraForSequenceClassification,
           "text-generation": ElectraForCausalLM,
           "token-classification": ElectraForTokenClassification,
           "zero-shot": ElectraForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )
   __A      :						Union[str, Any]   =							True






   def  __UpperCamelCase	( self							:	Optional[int]      ,		lowercase_							:	Optional[int]      ,		lowercase_							:	Optional[int]      ,		lowercase_							:	Any=False    )   ->		Optional[Any]:
       lowercase__					:  List[str]          =							super()._prepare_for_class(UpperCAmelCase__      ,		UpperCAmelCase__      ,		return_labels=UpperCAmelCase__    )

       if return_labels:
           if model_class in get_values(UpperCAmelCase__    ):
               lowercase__					:  Tuple          =							torch.zeros(
                   (self.model_tester.batch_size, self.model_tester.seq_length)      ,		dtype=torch.long      ,		device=UpperCAmelCase__    )
       return inputs_dict






   def  __UpperCamelCase	( self							:	Tuple    )   ->		int:
       lowercase__					:  Optional[Any]          =							ElectraModelTester(self    )
       lowercase__					:  List[str]          =							ConfigTester(self      ,		config_class=UpperCAmelCase__      ,		hidden_size=37    )






   def  __UpperCamelCase	( self							:	str    )   ->		List[Any]:
       self.config_tester.run_common_tests()






   def  __UpperCamelCase	( self							:	List[Any]    )   ->		Tuple:
       lowercase__					:  Union[str, Any]          =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_electra_model(*UpperCAmelCase__    )






   def  __UpperCamelCase	( self							:	Optional[int]    )   ->		List[str]:
       lowercase__					:  Optional[Any]          =							self.model_tester.prepare_config_and_inputs_for_decoder()
       self.model_tester.create_and_check_electra_model_as_decoder(*UpperCAmelCase__    )






   def  __UpperCamelCase	( self							:	Dict    )   ->		List[Any]:
       lowercase__					:  int          =							self.model_tester.prepare_config_and_inputs()
       for type in ["absolute", "relative_key", "relative_key_query"]:
           lowercase__					:  List[str]          =							type
           self.model_tester.create_and_check_electra_model(*UpperCAmelCase__    )






   def  __UpperCamelCase	( self							:	Optional[int]    )   ->		List[Any]:
       lowercase__					:  Optional[Any]          =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_electra_for_masked_lm(*UpperCAmelCase__    )






   def  __UpperCamelCase	( self							:	Union[str, Any]    )   ->		Any:
       lowercase__					:  Dict          =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_electra_for_token_classification(*UpperCAmelCase__    )






   def  __UpperCamelCase	( self							:	Union[str, Any]    )   ->		Union[str, Any]:
       lowercase__					:  List[str]          =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_electra_for_pretraining(*UpperCAmelCase__    )






   def  __UpperCamelCase	( self							:	Dict    )   ->		Tuple:
       lowercase__					:  Any          =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_electra_for_sequence_classification(*UpperCAmelCase__    )






   def  __UpperCamelCase	( self							:	List[Any]    )   ->		int:
       lowercase__					:  Optional[Any]          =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_electra_for_question_answering(*UpperCAmelCase__    )






   def  __UpperCamelCase	( self							:	str    )   ->		List[Any]:
       lowercase__					:  Optional[Any]          =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_electra_for_multiple_choice(*UpperCAmelCase__    )






   @slow
   def  __UpperCamelCase	( self							:	List[Any]    )   ->		List[str]:
       for model_name in ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           lowercase__					:  List[str]          =							ElectraModel.from_pretrained(UpperCAmelCase__    )
           self.assertIsNotNone(UpperCAmelCase__    )







   def  __UpperCamelCase	( self							:	Optional[Any]    )   ->		List[Any]:
       lowercase__					:  str          =							self.model_tester.prepare_config_and_inputs_for_decoder()
       self.model_tester.create_and_check_electra_for_causal_lm(*UpperCAmelCase__    )



@require_torch
class     snake_case_			(						unittest.TestCase			):






   @slow
   def  __UpperCamelCase	( self							:	Union[str, Any]    )   ->		int:
       lowercase__					:  Any          =							ElectraModel.from_pretrained("google/electra-small-discriminator"    )
       lowercase__					:  Union[str, Any]          =							torch.tensor([[0, 3_45, 2_32, 3_28, 7_40, 1_40, 16_95, 69, 60_78, 15_88, 2]]    )
       lowercase__					:  Union[str, Any]          =							torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]    )
       lowercase__					:  Optional[int]          =							model(UpperCAmelCase__      ,		attention_mask=UpperCAmelCase__    )[0]
       lowercase__					:  Tuple          =							torch.Size((1, 11, 2_56)    )
       self.assertEqual(output.shape      ,		UpperCAmelCase__    )
       lowercase__					:  Optional[Any]          =							torch.tensor(
           [[[0.44_71, 0.68_21, -0.32_65], [0.46_27, 0.52_55, -0.36_68], [0.45_32, 0.33_13, -0.43_44]]]    )

       self.assertTrue(torch.allclose(output[:, 1:4, 1:4]      ,		UpperCAmelCase__      ,		atol=1E-4    )    )










'''simple docstring'''






import argparse
import json
import re
import string
import sys
from collections import Counter





def      lowercase  (  lowerCAmelCase :						int):



				"""simple docstring"""


				def remove_articles(lowerCAmelCase :						List[Any]):
								return re.sub(R'''\b(a|an|the)\b'''   , ''' '''   , lowerCAmelCase)

				def white_space_fix(lowerCAmelCase :						Tuple):
								return " ".join(text.split())

				def remove_punc(lowerCAmelCase :						str):
								_A :				List[Any]				=  set(string.punctuation)
								return "".join(ch for ch in text if ch not in exclude)

				def lower(lowerCAmelCase :						Any):
								return text.lower()

				return white_space_fix(remove_articles(remove_punc(lower(lowerCAmelCase))))





def      lowercase  (  lowerCAmelCase :						int   , lowerCAmelCase :						Dict):



				"""simple docstring"""


				_A :				str				=  normalize_answer(lowerCAmelCase).split()
				_A :				Dict				=  normalize_answer(lowerCAmelCase).split()
				_A :				str				=  Counter(lowerCAmelCase) & Counter(lowerCAmelCase)
				_A :				List[Any]				=  sum(common.values())
				if num_same == 0:
								return 0
				_A :				Tuple				=  1.0 * num_same / len(lowerCAmelCase)
				_A :				List[Any]				=  1.0 * num_same / len(lowerCAmelCase)
				_A :				int				=  (2 * precision * recall) / (precision + recall)
				return fa





def      lowercase  (  lowerCAmelCase :						Any   , lowerCAmelCase :						str):



				"""simple docstring"""


				return normalize_answer(lowerCAmelCase) == normalize_answer(lowerCAmelCase)





def      lowercase  (  lowerCAmelCase :						Optional[Any]   , lowerCAmelCase :						Optional[int]   , lowerCAmelCase :						Optional[int]):



				"""simple docstring"""


				_A :				Optional[int]				=  []
				for ground_truth in ground_truths:
								_A :				Union[str, Any]				=  metric_fn(lowerCAmelCase   , lowerCAmelCase)
								scores_for_ground_truths.append(lowerCAmelCase)
				return max(lowerCAmelCase)





def      lowercase  (  lowerCAmelCase :						int   , lowerCAmelCase :						str):



				"""simple docstring"""


				_A :				Union[str, Any]				=  0
				_A :				List[Any]				=  []
				for passage in dataset:
								for qa in passage["qas"]:
												total += 1
												if qa["id"] not in predictions:
																_A :				str				=  f"""Unanswered question {qa['id']} will receive score 0."""
																print(lowerCAmelCase   , file=sys.stderr)
																continue

												_A :				Optional[int]				=  [x['''text'''] for x in qa['''answers''']]
												_A :				Union[str, Any]				=  predictions[qa['''id''']]

												_A :				Optional[Any]				=  metric_max_over_ground_truths(lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase)
												if int(_exact_match) == 1:
																correct_ids.append(qa['''id'''])
												exact_match += _exact_match

												fa += metric_max_over_ground_truths(lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase)

				_A :				Dict				=  exact_match / total
				_A :				Dict				=  fa / total

				return {"exact_match": exact_match, "f1": fa}, correct_ids


if __name__ == "__main__":
	__UpperCamelCase      :   List[str]								=     '''1.0'''
	__UpperCamelCase      :   Union[str, Any]								=     argparse.ArgumentParser('''Official evaluation script for ReCoRD v1.0.''')
	parser.add_argument('''data_file''', help='''The dataset file in JSON format.''')
	parser.add_argument('''pred_file''', help='''The model prediction file in JSON format.''')
	parser.add_argument('''--output_correct_ids''', action='''store_true''', help='''Output the correctly answered query IDs.''')
	__UpperCamelCase      :   Optional[int]								=     parser.parse_args()

	with open(args.data_file) as data_file:
		__UpperCamelCase      :   Optional[int]								=     json.load(data_file)
		if dataset_json["version"] != expected_version:
			print(
			    f'Evaluation expects v-{expected_version}, but got dataset with v-{dataset_json["version"]}',
			    file=sys.stderr,
			)
		__UpperCamelCase      :   Optional[Any]								=     dataset_json['''data''']

	with open(args.pred_file) as pred_file:
		__UpperCamelCase      :   int								=     json.load(pred_file)

	__UpperCamelCase							,  __UpperCamelCase      :   List[str]								=     evaluate(dataset, predictions)

	if args.output_correct_ids:
		print(f'Output {len(correct_ids)} correctly answered question IDs.')
		with open('''correct_ids.json''', '''w''') as f:
			json.dump(correct_ids, f)
