





'''simple docstring'''






import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class  lowerCamelCase__				(						snake_case_  ,							unittest.TestCase						):


	"""simple docstring"""





	__magic_name__							   =		ElectraTokenizer
	__magic_name__							   =		ElectraTokenizerFast
	__magic_name__							   =		True
	__magic_name__							   =		True
	__magic_name__							   =		filter_non_english




	def 				_lowerCamelCase				(					self			)       ->				int:
					super().setUp()

					_A :				Any				=  [
					    '''[UNK]''',
					    '''[CLS]''',
					    '''[SEP]''',
					    '''[PAD]''',
					    '''[MASK]''',
					    '''want''',
					    '''##want''',
					    '''##ed''',
					    '''wa''',
					    '''un''',
					    '''runn''',
					    '''##ing''',
					    ''',''',
					    '''low''',
					    '''lowest''',
					]
					_A :				str				=  os.path.join(self.tmpdirname   ,     VOCAB_FILES_NAMES['''vocab_file''']			)
					with open(self.vocab_file   ,     '''w'''   ,     encoding='''utf-8'''			) as vocab_writer:
									vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]			)			)




	def 				_lowerCamelCase				(					self   ,     UpperCAmelCase__			)       ->				str:
					_A :				Union[str, Any]				=  '''UNwant\u00E9d,running'''
					_A :				Dict				=  '''unwanted, running'''
					return input_text, output_text




	def 				_lowerCamelCase				(					self			)       ->				Optional[Any]:
					_A :				Tuple				=  self.tokenizer_class(self.vocab_file			)

					_A :				Union[str, Any]				=  tokenizer.tokenize('''UNwant\u00E9d,running'''			)
					self.assertListEqual(UpperCAmelCase__   ,     ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']			)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__			)   ,     [9, 6, 7, 1_2, 1_0, 1_1]			)




	def 				_lowerCamelCase				(					self			)       ->				Optional[int]:
					if not self.test_rust_tokenizer:
									return

					_A :				Tuple				=  self.get_tokenizer()
					_A :				Dict				=  self.get_rust_tokenizer()

					_A :				Dict				=  '''UNwant\u00E9d,running'''

					_A :				Optional[int]				=  tokenizer.tokenize(UpperCAmelCase__			)
					_A :				Optional[int]				=  rust_tokenizer.tokenize(UpperCAmelCase__			)
					self.assertListEqual(UpperCAmelCase__   ,     UpperCAmelCase__			)

					_A :				int				=  tokenizer.encode(UpperCAmelCase__   ,     add_special_tokens=UpperCAmelCase__			)
					_A :				Any				=  rust_tokenizer.encode(UpperCAmelCase__   ,     add_special_tokens=UpperCAmelCase__			)
					self.assertListEqual(UpperCAmelCase__   ,     UpperCAmelCase__			)

					_A :				Dict				=  self.get_rust_tokenizer()
					_A :				Optional[Any]				=  tokenizer.encode(UpperCAmelCase__			)
					_A :				int				=  rust_tokenizer.encode(UpperCAmelCase__			)
					self.assertListEqual(UpperCAmelCase__   ,     UpperCAmelCase__			)

					# With lower casing
					_A :				Dict				=  self.get_tokenizer(do_lower_case=UpperCAmelCase__			)
					_A :				List[Any]				=  self.get_rust_tokenizer(do_lower_case=UpperCAmelCase__			)

					_A :				Union[str, Any]				=  '''UNwant\u00E9d,running'''

					_A :				int				=  tokenizer.tokenize(UpperCAmelCase__			)
					_A :				List[Any]				=  rust_tokenizer.tokenize(UpperCAmelCase__			)
					self.assertListEqual(UpperCAmelCase__   ,     UpperCAmelCase__			)

					_A :				Any				=  tokenizer.encode(UpperCAmelCase__   ,     add_special_tokens=UpperCAmelCase__			)
					_A :				str				=  rust_tokenizer.encode(UpperCAmelCase__   ,     add_special_tokens=UpperCAmelCase__			)
					self.assertListEqual(UpperCAmelCase__   ,     UpperCAmelCase__			)

					_A :				Union[str, Any]				=  self.get_rust_tokenizer()
					_A :				Optional[int]				=  tokenizer.encode(UpperCAmelCase__			)
					_A :				int				=  rust_tokenizer.encode(UpperCAmelCase__			)
					self.assertListEqual(UpperCAmelCase__   ,     UpperCAmelCase__			)




	def 				_lowerCamelCase				(					self			)       ->				Optional[int]:
					_A :				Tuple				=  BasicTokenizer()

					self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''			)   ,     ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']			)




	def 				_lowerCamelCase				(					self			)       ->				Tuple:
					_A :				List[str]				=  BasicTokenizer(do_lower_case=UpperCAmelCase__			)

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''			)   ,     ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)   ,     ['''hello''']			)




	def 				_lowerCamelCase				(					self			)       ->				int:
					_A :				List[Any]				=  BasicTokenizer(do_lower_case=UpperCAmelCase__   ,     strip_accents=UpperCAmelCase__			)

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)   ,     ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)   ,     ['''h\u00E9llo''']			)




	def 				_lowerCamelCase				(					self			)       ->				Union[str, Any]:
					_A :				int				=  BasicTokenizer(do_lower_case=UpperCAmelCase__   ,     strip_accents=UpperCAmelCase__			)

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)   ,     ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)   ,     ['''hello''']			)




	def 				_lowerCamelCase				(					self			)       ->				Union[str, Any]:
					_A :				int				=  BasicTokenizer(do_lower_case=UpperCAmelCase__			)

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)   ,     ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)   ,     ['''hello''']			)




	def 				_lowerCamelCase				(					self			)       ->				Dict:
					_A :				Dict				=  BasicTokenizer(do_lower_case=UpperCAmelCase__			)

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''			)   ,     ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']			)




	def 				_lowerCamelCase				(					self			)       ->				Any:
					_A :				Optional[Any]				=  BasicTokenizer(do_lower_case=UpperCAmelCase__   ,     strip_accents=UpperCAmelCase__			)

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)   ,     ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']			)




	def 				_lowerCamelCase				(					self			)       ->				Any:
					_A :				Dict				=  BasicTokenizer(do_lower_case=UpperCAmelCase__   ,     strip_accents=UpperCAmelCase__			)

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)   ,     ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']			)




	def 				_lowerCamelCase				(					self			)       ->				Tuple:
					_A :				Dict				=  BasicTokenizer(do_lower_case=UpperCAmelCase__   ,     never_split=['''[UNK]''']			)

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''			)   ,     ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']			)




	def 				_lowerCamelCase				(					self			)       ->				List[Any]:
					_A :				str				=  ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

					_A :				Tuple				=  {}
					for i, token in enumerate(UpperCAmelCase__			):
									_A :				Tuple				=  i
					_A :				Optional[Any]				=  WordpieceTokenizer(vocab=UpperCAmelCase__   ,     unk_token='''[UNK]'''			)

					self.assertListEqual(tokenizer.tokenize(''''''			)   ,     []			)

					self.assertListEqual(tokenizer.tokenize('''unwanted running'''			)   ,     ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']			)

					self.assertListEqual(tokenizer.tokenize('''unwantedX running'''			)   ,     ['''[UNK]''', '''runn''', '''##ing''']			)




	def 				_lowerCamelCase				(					self			)       ->				Optional[int]:
					self.assertTrue(_is_whitespace(''' '''			)			)
					self.assertTrue(_is_whitespace('''\t'''			)			)
					self.assertTrue(_is_whitespace('''\r'''			)			)
					self.assertTrue(_is_whitespace('''\n'''			)			)
					self.assertTrue(_is_whitespace('''\u00A0'''			)			)

					self.assertFalse(_is_whitespace('''A'''			)			)
					self.assertFalse(_is_whitespace('''-'''			)			)




	def 				_lowerCamelCase				(					self			)       ->				Optional[int]:
					self.assertTrue(_is_control('''\u0005'''			)			)

					self.assertFalse(_is_control('''A'''			)			)
					self.assertFalse(_is_control(''' '''			)			)
					self.assertFalse(_is_control('''\t'''			)			)
					self.assertFalse(_is_control('''\r'''			)			)




	def 				_lowerCamelCase				(					self			)       ->				Dict:
					self.assertTrue(_is_punctuation('''-'''			)			)
					self.assertTrue(_is_punctuation('''$'''			)			)
					self.assertTrue(_is_punctuation('''`'''			)			)
					self.assertTrue(_is_punctuation('''.'''			)			)

					self.assertFalse(_is_punctuation('''A'''			)			)
					self.assertFalse(_is_punctuation(''' '''			)			)




	def 				_lowerCamelCase				(					self			)       ->				List[Any]:
					_A :				Tuple				=  self.get_tokenizer()
					_A :				Dict				=  self.get_rust_tokenizer()

					# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
					self.assertListEqual([tokenizer.tokenize(UpperCAmelCase__			) for t in ['''Test''', '''\xad''', '''test''']]   ,     [['''[UNK]'''], [], ['''[UNK]''']]			)

					self.assertListEqual(
					    [rust_tokenizer.tokenize(UpperCAmelCase__			) for t in ['''Test''', '''\xad''', '''test''']]   ,     [['''[UNK]'''], [], ['''[UNK]''']]			)




	@slow
	def 				_lowerCamelCase				(					self			)       ->				Any:
					_A :				Dict				=  self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''			)

					_A :				Optional[int]				=  tokenizer.encode('''sequence builders'''   ,     add_special_tokens=UpperCAmelCase__			)
					_A :				Union[str, Any]				=  tokenizer.encode('''multi-sequence build'''   ,     add_special_tokens=UpperCAmelCase__			)

					_A :				Optional[Any]				=  tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__			)
					_A :				Dict				=  tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__   ,     UpperCAmelCase__			)

					assert encoded_sentence == [1_0_1] + text + [1_0_2]
					assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]




	def 				_lowerCamelCase				(					self			)       ->				Dict:
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
													_A :				int				=  self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__   ,     **UpperCAmelCase__			)

													_A :				Optional[Any]				=  F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
													_A :				Union[str, Any]				=  tokenizer_r.encode_plus(
													    UpperCAmelCase__   ,     return_attention_mask=UpperCAmelCase__   ,     return_token_type_ids=UpperCAmelCase__   ,     return_offsets_mapping=UpperCAmelCase__   ,     add_special_tokens=UpperCAmelCase__   ,     )

													_A :				Tuple				=  tokenizer_r.do_lower_case if hasattr(UpperCAmelCase__   ,     '''do_lower_case'''			) else False
													_A :				Union[str, Any]				=  (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), '''A'''),
													        ((1, 2), ''','''),
													        ((3, 5), '''na'''),
													        ((5, 6), '''##ï'''),
													        ((6, 8), '''##ve'''),
													        ((9, 1_5), tokenizer_r.mask_token),
													        ((1_6, 2_1), '''Allen'''),
													        ((2_1, 2_3), '''##NL'''),
													        ((2_3, 2_4), '''##P'''),
													        ((2_5, 3_3), '''sentence'''),
													        ((3_3, 3_4), '''.'''),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), '''a'''),
													        ((1, 2), ''','''),
													        ((3, 8), '''naive'''),
													        ((9, 1_5), tokenizer_r.mask_token),
													        ((1_6, 2_1), '''allen'''),
													        ((2_1, 2_3), '''##nl'''),
													        ((2_3, 2_4), '''##p'''),
													        ((2_5, 3_3), '''sentence'''),
													        ((3_3, 3_4), '''.'''),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]   ,     tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']			)			)
													self.assertEqual([e[0] for e in expected_results]   ,     tokens['''offset_mapping''']			)




	def 				_lowerCamelCase				(					self			)       ->				int:
					_A :				str				=  ['''的''', '''人''', '''有''']
					_A :				Any				=  ''''''.join(UpperCAmelCase__			)
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
													_A :				Dict				=  True
													_A :				str				=  self.tokenizer_class.from_pretrained(UpperCAmelCase__   ,     **UpperCAmelCase__			)
													_A :				int				=  self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__   ,     **UpperCAmelCase__			)

													_A :				Dict				=  tokenizer_p.encode(UpperCAmelCase__   ,     add_special_tokens=UpperCAmelCase__			)
													_A :				Optional[Any]				=  tokenizer_r.encode(UpperCAmelCase__   ,     add_special_tokens=UpperCAmelCase__			)

													_A :				Any				=  tokenizer_r.convert_ids_to_tokens(UpperCAmelCase__			)
													_A :				Union[str, Any]				=  tokenizer_p.convert_ids_to_tokens(UpperCAmelCase__			)

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(UpperCAmelCase__   ,     UpperCAmelCase__			)
													self.assertListEqual(UpperCAmelCase__   ,     UpperCAmelCase__			)

													_A :				Tuple				=  False
													_A :				List[Any]				=  self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__   ,     **UpperCAmelCase__			)
													_A :				Optional[int]				=  self.tokenizer_class.from_pretrained(UpperCAmelCase__   ,     **UpperCAmelCase__			)

													_A :				Optional[int]				=  tokenizer_r.encode(UpperCAmelCase__   ,     add_special_tokens=UpperCAmelCase__			)
													_A :				Tuple				=  tokenizer_p.encode(UpperCAmelCase__   ,     add_special_tokens=UpperCAmelCase__			)

													_A :				Dict				=  tokenizer_r.convert_ids_to_tokens(UpperCAmelCase__			)
													_A :				Tuple				=  tokenizer_p.convert_ids_to_tokens(UpperCAmelCase__			)

													# it is expected that only the first Chinese character is not preceded by "##".
													_A :				Optional[Any]				=  [
													    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(UpperCAmelCase__			)
													]
													self.assertListEqual(UpperCAmelCase__   ,     UpperCAmelCase__			)
													self.assertListEqual(UpperCAmelCase__   ,     UpperCAmelCase__			)







'''simple docstring'''






import argparse
import json
import re
import string
import sys
from collections import Counter





def      lowercase  (  lowerCAmelCase :						int):



				"""simple docstring"""


				def remove_articles(lowerCAmelCase :						List[Any]):
								return re.sub(R'''\b(a|an|the)\b'''   , ''' '''   , lowerCAmelCase)

				def white_space_fix(lowerCAmelCase :						Tuple):
								return " ".join(text.split())

				def remove_punc(lowerCAmelCase :						str):
								_A :				List[Any]				=  set(string.punctuation)
								return "".join(ch for ch in text if ch not in exclude)

				def lower(lowerCAmelCase :						Any):
								return text.lower()

				return white_space_fix(remove_articles(remove_punc(lower(lowerCAmelCase))))





def      lowercase  (  lowerCAmelCase :						int   , lowerCAmelCase :						Dict):



				"""simple docstring"""


				_A :				str				=  normalize_answer(lowerCAmelCase).split()
				_A :				Dict				=  normalize_answer(lowerCAmelCase).split()
				_A :				str				=  Counter(lowerCAmelCase) & Counter(lowerCAmelCase)
				_A :				List[Any]				=  sum(common.values())
				if num_same == 0:
								return 0
				_A :				Tuple				=  1.0 * num_same / len(lowerCAmelCase)
				_A :				List[Any]				=  1.0 * num_same / len(lowerCAmelCase)
				_A :				int				=  (2 * precision * recall) / (precision + recall)
				return fa





def      lowercase  (  lowerCAmelCase :						Any   , lowerCAmelCase :						str):



				"""simple docstring"""


				return normalize_answer(lowerCAmelCase) == normalize_answer(lowerCAmelCase)





def      lowercase  (  lowerCAmelCase :						Optional[Any]   , lowerCAmelCase :						Optional[int]   , lowerCAmelCase :						Optional[int]):



				"""simple docstring"""


				_A :				Optional[int]				=  []
				for ground_truth in ground_truths:
								_A :				Union[str, Any]				=  metric_fn(lowerCAmelCase   , lowerCAmelCase)
								scores_for_ground_truths.append(lowerCAmelCase)
				return max(lowerCAmelCase)





def      lowercase  (  lowerCAmelCase :						int   , lowerCAmelCase :						str):



				"""simple docstring"""


				_A :				Union[str, Any]				=  0
				_A :				List[Any]				=  []
				for passage in dataset:
								for qa in passage["qas"]:
												total += 1
												if qa["id"] not in predictions:
																_A :				str				=  f"""Unanswered question {qa['id']} will receive score 0."""
																print(lowerCAmelCase   , file=sys.stderr)
																continue

												_A :				Optional[int]				=  [x['''text'''] for x in qa['''answers''']]
												_A :				Union[str, Any]				=  predictions[qa['''id''']]

												_A :				Optional[Any]				=  metric_max_over_ground_truths(lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase)
												if int(_exact_match) == 1:
																correct_ids.append(qa['''id'''])
												exact_match += _exact_match

												fa += metric_max_over_ground_truths(lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase)

				_A :				Dict				=  exact_match / total
				_A :				Dict				=  fa / total

				return {"exact_match": exact_match, "f1": fa}, correct_ids


if __name__ == "__main__":
	__UpperCamelCase      :   List[str]								=     '''1.0'''
	__UpperCamelCase      :   Union[str, Any]								=     argparse.ArgumentParser('''Official evaluation script for ReCoRD v1.0.''')
	parser.add_argument('''data_file''', help='''The dataset file in JSON format.''')
	parser.add_argument('''pred_file''', help='''The model prediction file in JSON format.''')
	parser.add_argument('''--output_correct_ids''', action='''store_true''', help='''Output the correctly answered query IDs.''')
	__UpperCamelCase      :   Optional[int]								=     parser.parse_args()

	with open(args.data_file) as data_file:
		__UpperCamelCase      :   Optional[int]								=     json.load(data_file)
		if dataset_json["version"] != expected_version:
			print(
			    f'Evaluation expects v-{expected_version}, but got dataset with v-{dataset_json["version"]}',
			    file=sys.stderr,
			)
		__UpperCamelCase      :   Optional[Any]								=     dataset_json['''data''']

	with open(args.pred_file) as pred_file:
		__UpperCamelCase      :   int								=     json.load(pred_file)

	__UpperCamelCase							,  __UpperCamelCase      :   List[str]								=     evaluate(dataset, predictions)

	if args.output_correct_ids:
		print(f'Output {len(correct_ids)} correctly answered question IDs.')
		with open('''correct_ids.json''', '''w''') as f:
			json.dump(correct_ids, f)
