






"""simple docstring"""





import argparse
import json
import re
import string
import sys
from collections import Counter






def 							A(    snake_case_						):
       """simple docstring"""





       def remove_articles(snake_case_						):
              return re.sub(r"\b(a|an|the)\b"		,  " "		,  snake_case_						)

       def white_space_fix(snake_case_						):
              return " ".join(text.split()						)

       def remove_punc(snake_case_						):
              lowercase__:							str         =     set(string.punctuation						)
              return "".join(ch for ch in text if ch not in exclude						)

       def lower(snake_case_						):
              return text.lower()

       return white_space_fix(remove_articles(remove_punc(lower(snake_case_						)						)						)						)






def 							A(    snake_case_		,  snake_case_						):
       """simple docstring"""





       lowercase__:							Dict         =     normalize_answer(snake_case_						).split()
       lowercase__:							List[str]         =     normalize_answer(snake_case_						).split()
       lowercase__:							Optional[int]         =     Counter(snake_case_						) & Counter(snake_case_						)
       lowercase__:							Any         =     sum(common.values()						)
       if num_same == 0:
              return 0
       lowercase__:							Any         =     1.0 * num_same / len(snake_case_						)
       lowercase__:							List[Any]         =     1.0 * num_same / len(snake_case_						)
       lowercase__:							int         =     (2 * precision * recall) / (precision + recall)
       return fa






def 							A(    snake_case_		,  snake_case_						):
       """simple docstring"""





       return normalize_answer(snake_case_						) == normalize_answer(snake_case_						)






def 							A(    snake_case_		,  snake_case_		,  snake_case_						):
       """simple docstring"""





       lowercase__:							int         =     []
       for ground_truth in ground_truths:
              lowercase__:							str         =     metric_fn(snake_case_		,  snake_case_						)
              scores_for_ground_truths.append(snake_case_						)
       return max(snake_case_						)






def 							A(    snake_case_		,  snake_case_						):
       """simple docstring"""





       lowercase__:							Optional[int]         =     0
       for article in dataset:
              for paragraph in article["paragraphs"]:
                     for qa in paragraph["qas"]:
                            total += 1
                            if qa["id"] not in predictions:
                                   lowercase__:							Union[str, Any]         =     "Unanswered question " + qa["id"] + " will receive score 0."
                                   print(snake_case_		,  file=sys.stderr						)
                                   continue
                            lowercase__:							List[Any]         =     [x["text"] for x in qa["answers"]]
                            lowercase__:							Optional[Any]         =     predictions[qa["id"]]
                            exact_match += metric_max_over_ground_truths(snake_case_		,  snake_case_		,  snake_case_						)
                            fa += metric_max_over_ground_truths(snake_case_		,  snake_case_		,  snake_case_						)

       lowercase__:							Union[str, Any]         =     1_00.0 * exact_match / total
       lowercase__:							str         =     1_00.0 * fa / total

       return {"exact_match": exact_match, "f1": fa}


if __name__ == "__main__":
  UpperCamelCase					=     """1.1"""
  UpperCamelCase					=     argparse.ArgumentParser(description="""Evaluation for SQuAD """ + expected_version)
  parser.add_argument("""dataset_file""", help="""Dataset file""")
  parser.add_argument("""prediction_file""", help="""Prediction File""")
  UpperCamelCase					=     parser.parse_args()
  with open(args.dataset_file) as dataset_file:
    UpperCamelCase					=     json.load(dataset_file)
    if dataset_json["version"] != expected_version:
      print(
          """Evaluation expects v-""" + expected_version + """, but got dataset with v-""" + dataset_json["""version"""],
          file=sys.stderr,
      )
    UpperCamelCase					=     dataset_json["""data"""]
  with open(args.prediction_file) as prediction_file:
    UpperCamelCase					=     json.load(prediction_file)
  print(json.dumps(evaluate(dataset, predictions)))














import unittest

from transformers import MarkupLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    MarkupLMForQuestionAnswering,
				    MarkupLMForSequenceClassification,
				    MarkupLMForTokenClassification,
				    MarkupLMModel,
				)

# TODO check dependencies
from transformers import MarkupLMFeatureExtractor, MarkupLMProcessor, MarkupLMTokenizer


class 				__a				:





		'''simple docstring'''



		def __init__(    self			,     UpperCamelCase__			,     UpperCamelCase__=13			,     UpperCamelCase__=7			,     UpperCamelCase__=True			,     UpperCamelCase__=True			,     UpperCamelCase__=True			,     UpperCamelCase__=True			,     UpperCamelCase__=99			,     UpperCamelCase__=32			,     UpperCamelCase__=5			,     UpperCamelCase__=4			,     UpperCamelCase__=37			,     UpperCamelCase__="gelu"			,     UpperCamelCase__=0.1			,     UpperCamelCase__=0.1			,     UpperCamelCase__=512			,     UpperCamelCase__=16			,     UpperCamelCase__=2			,     UpperCamelCase__=0.02			,     UpperCamelCase__=3			,     UpperCamelCase__=None			,     UpperCamelCase__=20			,     UpperCamelCase__=30			,     UpperCamelCase__=2			,     UpperCamelCase__=2			,     UpperCamelCase__=10			,     ):
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= parent
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= batch_size
						SCREAMING_SNAKE_CASE_ :						int    				= seq_length
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= is_training
						SCREAMING_SNAKE_CASE_ :						Dict    				= use_input_mask
						SCREAMING_SNAKE_CASE_ :						Dict    				= use_token_type_ids
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= use_labels
						SCREAMING_SNAKE_CASE_ :						int    				= vocab_size
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= hidden_size
						SCREAMING_SNAKE_CASE_ :						Any    				= num_hidden_layers
						SCREAMING_SNAKE_CASE_ :						int    				= num_attention_heads
						SCREAMING_SNAKE_CASE_ :						Any    				= intermediate_size
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= hidden_act
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= hidden_dropout_prob
						SCREAMING_SNAKE_CASE_ :						List[str]    				= attention_probs_dropout_prob
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= max_position_embeddings
						SCREAMING_SNAKE_CASE_ :						Tuple    				= type_vocab_size
						SCREAMING_SNAKE_CASE_ :						str    				= type_sequence_label_size
						SCREAMING_SNAKE_CASE_ :						Dict    				= initializer_range
						SCREAMING_SNAKE_CASE_ :						int    				= num_labels
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= scope
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= max_xpath_tag_unit_embeddings
						SCREAMING_SNAKE_CASE_ :						Tuple    				= max_xpath_subs_unit_embeddings
						SCREAMING_SNAKE_CASE_ :						Dict    				= tag_pad_id
						SCREAMING_SNAKE_CASE_ :						str    				= subs_pad_id
						SCREAMING_SNAKE_CASE_ :						Tuple    				= max_depth


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= ids_tensor([self.batch_size, self.seq_length]			,     self.vocab_size							)

						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= ids_tensor(
						    [self.batch_size, self.seq_length, self.max_depth]			,     self.max_xpath_tag_unit_embeddings							)

						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= ids_tensor(
						    [self.batch_size, self.seq_length, self.max_depth]			,     self.max_xpath_subs_unit_embeddings							)

						SCREAMING_SNAKE_CASE_ :						Any    				= None
						if self.use_input_mask:
										SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= ids_tensor([self.batch_size, self.seq_length]			,     vocab_size=2							)

						SCREAMING_SNAKE_CASE_ :						Dict    				= None
						if self.use_token_type_ids:
										SCREAMING_SNAKE_CASE_ :						int    				= ids_tensor([self.batch_size, self.seq_length]			,     self.type_vocab_size							)

						SCREAMING_SNAKE_CASE_ :						int    				= None
						SCREAMING_SNAKE_CASE_ :						Tuple    				= None
						if self.use_labels:
										SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= ids_tensor([self.batch_size]			,     self.type_sequence_label_size							)
										SCREAMING_SNAKE_CASE_ :						Any    				= ids_tensor([self.batch_size, self.seq_length]			,     self.num_labels							)

						SCREAMING_SNAKE_CASE_ :						Tuple    				= self.get_config()

						return (
						    config,
						    input_ids,
						    xpath_tags_seq,
						    xpath_subs_seq,
						    token_type_ids,
						    input_mask,
						    sequence_labels,
						    token_labels,
						)


		def 	__snake_case      (    self							):
						return MarkupLMConfig(
						    vocab_size=self.vocab_size			,     hidden_size=self.hidden_size			,     num_hidden_layers=self.num_hidden_layers			,     num_attention_heads=self.num_attention_heads			,     intermediate_size=self.intermediate_size			,     hidden_act=self.hidden_act			,     hidden_dropout_prob=self.hidden_dropout_prob			,     attention_probs_dropout_prob=self.attention_probs_dropout_prob			,     max_position_embeddings=self.max_position_embeddings			,     type_vocab_size=self.type_vocab_size			,     initializer_range=self.initializer_range			,     max_xpath_tag_unit_embeddings=self.max_xpath_tag_unit_embeddings			,     max_xpath_subs_unit_embeddings=self.max_xpath_subs_unit_embeddings			,     tag_pad_id=self.tag_pad_id			,     subs_pad_id=self.subs_pad_id			,     max_depth=self.max_depth			,     )


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     ):
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= MarkupLMModel(config=UpperCamelCase__							)
						model.to(UpperCamelCase__							)
						model.eval()
						print('Configs:'			,     model.config.tag_pad_id			,     model.config.subs_pad_id							)
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= model(UpperCamelCase__			,     attention_mask=UpperCamelCase__			,     token_type_ids=UpperCamelCase__							)
						SCREAMING_SNAKE_CASE_ :						List[str]    				= model(UpperCamelCase__			,     token_type_ids=UpperCamelCase__							)
						SCREAMING_SNAKE_CASE_ :						int    				= model(UpperCamelCase__							)
						self.parent.assertEqual(result.last_hidden_state.shape			,     (self.batch_size, self.seq_length, self.hidden_size)							)
						self.parent.assertEqual(result.pooler_output.shape			,     (self.batch_size, self.hidden_size)							)


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     ):
						SCREAMING_SNAKE_CASE_ :						Dict    				= self.num_labels
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= MarkupLMForSequenceClassification(UpperCamelCase__							)
						model.to(UpperCamelCase__							)
						model.eval()
						SCREAMING_SNAKE_CASE_ :						str    				= model(
						    UpperCamelCase__			,     xpath_tags_seq=UpperCamelCase__			,     xpath_subs_seq=UpperCamelCase__			,     attention_mask=UpperCamelCase__			,     token_type_ids=UpperCamelCase__			,     labels=UpperCamelCase__			,     )
						self.parent.assertEqual(result.logits.shape			,     (self.batch_size, self.num_labels)							)


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     ):
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= self.num_labels
						SCREAMING_SNAKE_CASE_ :						List[str]    				= MarkupLMForTokenClassification(config=UpperCamelCase__							)
						model.to(UpperCamelCase__							)
						model.eval()
						SCREAMING_SNAKE_CASE_ :						str    				= model(
						    UpperCamelCase__			,     xpath_tags_seq=UpperCamelCase__			,     xpath_subs_seq=UpperCamelCase__			,     attention_mask=UpperCamelCase__			,     token_type_ids=UpperCamelCase__			,     labels=UpperCamelCase__			,     )
						self.parent.assertEqual(result.logits.shape			,     (self.batch_size, self.seq_length, self.num_labels)							)


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     ):
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= MarkupLMForQuestionAnswering(config=UpperCamelCase__							)
						model.to(UpperCamelCase__							)
						model.eval()
						SCREAMING_SNAKE_CASE_ :						int    				= model(
						    UpperCamelCase__			,     xpath_tags_seq=UpperCamelCase__			,     xpath_subs_seq=UpperCamelCase__			,     attention_mask=UpperCamelCase__			,     token_type_ids=UpperCamelCase__			,     start_positions=UpperCamelCase__			,     end_positions=UpperCamelCase__			,     )
						self.parent.assertEqual(result.start_logits.shape			,     (self.batch_size, self.seq_length)							)
						self.parent.assertEqual(result.end_logits.shape			,     (self.batch_size, self.seq_length)							)






		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Any    				= self.prepare_config_and_inputs()
						(
						    (
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							
						) :						int    				= config_and_inputs
						SCREAMING_SNAKE_CASE_ :						Any    				= {
						    'input_ids': input_ids,
						    'xpath_tags_seq': xpath_tags_seq,
						    'xpath_subs_seq': xpath_subs_seq,
						    'token_type_ids': token_type_ids,
						    'attention_mask': input_mask,
						}
						return config, inputs_dict






@require_torch
class 				__a				(	__A			,						__A			,						unittest.TestCase    ):





		'''simple docstring'''



		UpperCAmelCase__   :					Dict					     =    (
		    (
		        MarkupLMModel,
		        MarkupLMForSequenceClassification,
		        MarkupLMForTokenClassification,
		        MarkupLMForQuestionAnswering,
		    )
		    if is_torch_available()
		    else None
		)
		UpperCAmelCase__   :					Union[str, Any]					     =    (
		    {
		        """feature-extraction""": MarkupLMModel,
		        """question-answering""": MarkupLMForQuestionAnswering,
		        """text-classification""": MarkupLMForSequenceClassification,
		        """token-classification""": MarkupLMForTokenClassification,
		        """zero-shot""": MarkupLMForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__							):
						# ValueError: Nodes must be of type `List[str]` (single pretokenized example), or `List[List[str]]`
						# (batch of pretokenized examples).
						return True


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						int    				= MarkupLMModelTester(self							)
						SCREAMING_SNAKE_CASE_ :						Any    				= ConfigTester(self			,     config_class=UpperCamelCase__			,     hidden_size=37							)


		def 	__snake_case      (    self							):
						self.config_tester.run_common_tests()


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*UpperCamelCase__							)


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_sequence_classification(*UpperCamelCase__							)


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Any    				= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_token_classification(*UpperCamelCase__							)






		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_question_answering(*UpperCamelCase__							)






def        _lowerCamelCase(      ):





				'''simple docstring'''







				SCREAMING_SNAKE_CASE_ :						int    				= '\n    <!DOCTYPE html>\n    <html>\n    <head>\n    <title>Page Title</title>\n    </head>\n    <body>\n\n    <h1>This is a Heading</h1>\n    <p>This is a paragraph.</p>\n\n    </body>\n    </html>\n    '

				return html_string


@require_torch
class 				__a				(	unittest.TestCase    ):





		'''simple docstring'''



		@cached_property
		def 	__snake_case      (    self							):
						# TODO use from_pretrained here
						SCREAMING_SNAKE_CASE_ :						List[str]    				= MarkupLMFeatureExtractor()
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= MarkupLMTokenizer.from_pretrained('microsoft/markuplm-base'							)

						return MarkupLMProcessor(UpperCamelCase__			,     UpperCamelCase__							)






		@slow
		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Tuple    				= MarkupLMModel.from_pretrained('microsoft/markuplm-base'							).to(UpperCamelCase__							)

						SCREAMING_SNAKE_CASE_ :						Tuple    				= self.default_processor

						SCREAMING_SNAKE_CASE_ :						int    				= processor(prepare_html_string()			,     return_tensors='pt'							)
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= inputs.to(UpperCamelCase__							)

						# forward pass
						with torch.no_grad():
										SCREAMING_SNAKE_CASE_ :						str    				= model(**UpperCamelCase__							)

						# verify the last hidden states
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= torch.Size([1, 14, 768]							)
						self.assertEqual(outputs.last_hidden_state.shape			,     UpperCamelCase__							)

						SCREAMING_SNAKE_CASE_ :						Tuple    				= torch.tensor(
						    [[0.06_75, -0.00_52, 0.50_01], [-0.22_81, 0.08_02, 0.21_92], [-0.05_83, -0.33_11, 0.11_85]]							).to(UpperCamelCase__							)

						self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]			,     UpperCamelCase__			,     atol=1E-4							)							)