

"""simple docstring"""





import unittest

from transformers import LayoutLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    LayoutLMForMaskedLM,
				    LayoutLMForQuestionAnswering,
				    LayoutLMForSequenceClassification,
				    LayoutLMForTokenClassification,
				    LayoutLMModel,
				)

class      __a :

			"""simple docstring"""

			def __init__(				self  :       Dict    ,	lowercase_  :       List[str]    ,	lowercase_  :       Dict=13    ,	lowercase_  :       Dict=7    ,	lowercase_  :       List[str]=True    ,	lowercase_  :       Any=True    ,	lowercase_  :       Union[str, Any]=True    ,	lowercase_  :       str=True    ,	lowercase_  :       Optional[int]=99    ,	lowercase_  :       Optional[Any]=32    ,	lowercase_  :       List[str]=5    ,	lowercase_  :       Union[str, Any]=4    ,	lowercase_  :       Tuple=37    ,	lowercase_  :       int="gelu"    ,	lowercase_  :       Union[str, Any]=0.1    ,	lowercase_  :       Dict=0.1    ,	lowercase_  :       Optional[int]=512    ,	lowercase_  :       Tuple=16    ,	lowercase_  :       int=2    ,	lowercase_  :       List[Any]=0.0_2    ,	lowercase_  :       int=3    ,	lowercase_  :       List[str]=4    ,	lowercase_  :       int=None    ,	lowercase_  :       int=1000    ,	):
								UpperCamelCase__		:     Optional[Any]								=parent
								UpperCamelCase__		:     List[Any]								=batch_size
								UpperCamelCase__		:     Any								=seq_length
								UpperCamelCase__		:     List[str]								=is_training
								UpperCamelCase__		:     Any								=use_input_mask
								UpperCamelCase__		:     Dict								=use_token_type_ids
								UpperCamelCase__		:     Union[str, Any]								=use_labels
								UpperCamelCase__		:     int								=vocab_size
								UpperCamelCase__		:     List[Any]								=hidden_size
								UpperCamelCase__		:     List[str]								=num_hidden_layers
								UpperCamelCase__		:     str								=num_attention_heads
								UpperCamelCase__		:     str								=intermediate_size
								UpperCamelCase__		:     Optional[int]								=hidden_act
								UpperCamelCase__		:     Tuple								=hidden_dropout_prob
								UpperCamelCase__		:     List[str]								=attention_probs_dropout_prob
								UpperCamelCase__		:     List[Any]								=max_position_embeddings
								UpperCamelCase__		:     Optional[Any]								=type_vocab_size
								UpperCamelCase__		:     Optional[int]								=type_sequence_label_size
								UpperCamelCase__		:     Tuple								=initializer_range
								UpperCamelCase__		:     Optional[int]								=num_labels
								UpperCamelCase__		:     List[str]								=num_choices
								UpperCamelCase__		:     List[str]								=scope
								UpperCamelCase__		:     Dict								=range_bbox






			def  _lowerCAmelCase      (				self  :       str    ):
								UpperCamelCase__		:     Optional[int]								=ids_tensor([self.batch_size, self.seq_length]    ,	self.vocab_size    )

								UpperCamelCase__		:     List[str]								=ids_tensor([self.batch_size, self.seq_length, 4]    ,	self.range_bbox    )
								# Ensure that bbox is legal
								for i in range(bbox.shape[0]    ):
													for j in range(bbox.shape[1]    ):
																		if bbox[i, j, 3] < bbox[i, j, 1]:
																							UpperCamelCase__		:     List[Any]								=bbox[i, j, 3]
																							UpperCamelCase__		:     Optional[Any]								=bbox[i, j, 1]
																							UpperCamelCase__		:     Tuple								=t
																		if bbox[i, j, 2] < bbox[i, j, 0]:
																							UpperCamelCase__		:     str								=bbox[i, j, 2]
																							UpperCamelCase__		:     List[Any]								=bbox[i, j, 0]
																							UpperCamelCase__		:     Any								=t

								UpperCamelCase__		:     Optional[Any]								=None
								if self.use_input_mask:
													UpperCamelCase__		:     str								=ids_tensor([self.batch_size, self.seq_length]    ,	vocab_size=2    )

								UpperCamelCase__		:     str								=None
								if self.use_token_type_ids:
													UpperCamelCase__		:     List[str]								=ids_tensor([self.batch_size, self.seq_length]    ,	self.type_vocab_size    )

								UpperCamelCase__		:     Optional[int]								=None
								UpperCamelCase__		:     Any								=None
								UpperCamelCase__		:     List[str]								=None
								if self.use_labels:
													UpperCamelCase__		:     str								=ids_tensor([self.batch_size]    ,	self.type_sequence_label_size    )
													UpperCamelCase__		:     Union[str, Any]								=ids_tensor([self.batch_size, self.seq_length]    ,	self.num_labels    )
													UpperCamelCase__		:     List[Any]								=ids_tensor([self.batch_size]    ,	self.num_choices    )

								UpperCamelCase__		:     Optional[Any]								=self.get_config()

								return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






			def  _lowerCAmelCase      (				self  :       Any    ):
								return LayoutLMConfig(
								    vocab_size=self.vocab_size    ,	hidden_size=self.hidden_size    ,	num_hidden_layers=self.num_hidden_layers    ,	num_attention_heads=self.num_attention_heads    ,	intermediate_size=self.intermediate_size    ,	hidden_act=self.hidden_act    ,	hidden_dropout_prob=self.hidden_dropout_prob    ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,	max_position_embeddings=self.max_position_embeddings    ,	type_vocab_size=self.type_vocab_size    ,	initializer_range=self.initializer_range    ,	)






			def  _lowerCAmelCase      (				self  :       Optional[int]    ,	lowercase_  :       Any    ,	lowercase_  :       int    ,	lowercase_  :       List[Any]    ,	lowercase_  :       Union[str, Any]    ,	lowercase_  :       Optional[int]    ,	lowercase_  :       int    ,	lowercase_  :       Optional[int]    ,	lowercase_  :       Dict    ):
								UpperCamelCase__		:     str								=LayoutLMModel(config=lowercase_    )
								model.to(lowercase_    )
								model.eval()
								UpperCamelCase__		:     List[Any]								=model(lowercase_    ,	lowercase_    ,	attention_mask=lowercase_    ,	token_type_ids=lowercase_    )
								UpperCamelCase__		:     List[str]								=model(lowercase_    ,	lowercase_    ,	token_type_ids=lowercase_    )
								UpperCamelCase__		:     Any								=model(lowercase_    ,	lowercase_    )
								self.parent.assertEqual(result.last_hidden_state.shape    ,	(self.batch_size, self.seq_length, self.hidden_size)    )
								self.parent.assertEqual(result.pooler_output.shape    ,	(self.batch_size, self.hidden_size)    )






			def  _lowerCAmelCase      (				self  :       List[Any]    ,	lowercase_  :       int    ,	lowercase_  :       Optional[Any]    ,	lowercase_  :       Optional[Any]    ,	lowercase_  :       Tuple    ,	lowercase_  :       Tuple    ,	lowercase_  :       Tuple    ,	lowercase_  :       Optional[int]    ,	lowercase_  :       int    ):
								UpperCamelCase__		:     int								=LayoutLMForMaskedLM(config=lowercase_    )
								model.to(lowercase_    )
								model.eval()
								UpperCamelCase__		:     Dict								=model(lowercase_    ,	lowercase_    ,	attention_mask=lowercase_    ,	token_type_ids=lowercase_    ,	labels=lowercase_    )
								self.parent.assertEqual(result.logits.shape    ,	(self.batch_size, self.seq_length, self.vocab_size)    )






			def  _lowerCAmelCase      (				self  :       Union[str, Any]    ,	lowercase_  :       Dict    ,	lowercase_  :       Tuple    ,	lowercase_  :       Dict    ,	lowercase_  :       List[Any]    ,	lowercase_  :       Any    ,	lowercase_  :       Union[str, Any]    ,	lowercase_  :       str    ,	lowercase_  :       Dict    ):
								UpperCamelCase__		:     Optional[Any]								=self.num_labels
								UpperCamelCase__		:     Union[str, Any]								=LayoutLMForSequenceClassification(lowercase_    )
								model.to(lowercase_    )
								model.eval()
								UpperCamelCase__		:     Optional[int]								=model(
								    lowercase_    ,	lowercase_    ,	attention_mask=lowercase_    ,	token_type_ids=lowercase_    ,	labels=lowercase_    )
								self.parent.assertEqual(result.logits.shape    ,	(self.batch_size, self.num_labels)    )






			def  _lowerCAmelCase      (				self  :       Dict    ,	lowercase_  :       List[Any]    ,	lowercase_  :       List[str]    ,	lowercase_  :       Union[str, Any]    ,	lowercase_  :       int    ,	lowercase_  :       Any    ,	lowercase_  :       Any    ,	lowercase_  :       Optional[int]    ,	lowercase_  :       int    ):
								UpperCamelCase__		:     str								=self.num_labels
								UpperCamelCase__		:     Optional[int]								=LayoutLMForTokenClassification(config=lowercase_    )
								model.to(lowercase_    )
								model.eval()
								UpperCamelCase__		:     int								=model(lowercase_    ,	lowercase_    ,	attention_mask=lowercase_    ,	token_type_ids=lowercase_    ,	labels=lowercase_    )
								self.parent.assertEqual(result.logits.shape    ,	(self.batch_size, self.seq_length, self.num_labels)    )






			def  _lowerCAmelCase      (				self  :       int    ,	lowercase_  :       str    ,	lowercase_  :       List[Any]    ,	lowercase_  :       List[str]    ,	lowercase_  :       List[str]    ,	lowercase_  :       Union[str, Any]    ,	lowercase_  :       Optional[int]    ,	lowercase_  :       List[str]    ,	lowercase_  :       Optional[Any]    ):
								UpperCamelCase__		:     Any								=LayoutLMForQuestionAnswering(config=lowercase_    )
								model.to(lowercase_    )
								model.eval()
								UpperCamelCase__		:     List[Any]								=model(
								    lowercase_    ,	bbox=lowercase_    ,	attention_mask=lowercase_    ,	token_type_ids=lowercase_    ,	start_positions=lowercase_    ,	end_positions=lowercase_    ,	)
								self.parent.assertEqual(result.start_logits.shape    ,	(self.batch_size, self.seq_length)    )
								self.parent.assertEqual(result.end_logits.shape    ,	(self.batch_size, self.seq_length)    )






			def  _lowerCAmelCase      (				self  :       Optional[Any]    ):
								UpperCamelCase__		:     List[Any]								=self.prepare_config_and_inputs()
								(
								    (
								    UpperCamelCase__
								)	,      (
								    UpperCamelCase__
								)	,      (
								    UpperCamelCase__
								)	,      (
								    UpperCamelCase__
								)	,      (
								    UpperCamelCase__
								)	,      (
								    UpperCamelCase__
								)	,      (
								    UpperCamelCase__
								)	,      (
								    UpperCamelCase__
								)	,      
								)		:     Optional[int]								=config_and_inputs
								UpperCamelCase__		:     Union[str, Any]								={
								    '''input_ids''': input_ids,
								    '''bbox''': bbox,
								    '''token_type_ids''': token_type_ids,
								    '''attention_mask''': input_mask,
								}
								return config, inputs_dict



@require_torch
class      __a (   snake_case__,     snake_case__,     unittest.TestCase     ):

			"""simple docstring"""

			SCREAMING_SNAKE_CASE_      =    (
			    (
			        LayoutLMModel,
			        LayoutLMForMaskedLM,
			        LayoutLMForSequenceClassification,
			        LayoutLMForTokenClassification,
			        LayoutLMForQuestionAnswering,
			    )
			    if is_torch_available()
			    else None
			)
			SCREAMING_SNAKE_CASE_      =    (
			    {
			        'document-question-answering': LayoutLMForQuestionAnswering,
			        'feature-extraction': LayoutLMModel,
			        'fill-mask': LayoutLMForMaskedLM,
			        'text-classification': LayoutLMForSequenceClassification,
			        'token-classification': LayoutLMForTokenClassification,
			        'zero-shot': LayoutLMForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			SCREAMING_SNAKE_CASE_      =    True






			def  _lowerCAmelCase      (				self  :       List[str]    ):
								UpperCamelCase__		:     List[Any]								=LayoutLMModelTester(self    )
								UpperCamelCase__		:     List[str]								=ConfigTester(self    ,	config_class=lowercase_    ,	hidden_size=37    )






			def  _lowerCAmelCase      (				self  :       List[str]    ):
								self.config_tester.run_common_tests()






			def  _lowerCAmelCase      (				self  :       Any    ):
								UpperCamelCase__		:     List[Any]								=self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*lowercase_    )






			def  _lowerCAmelCase      (				self  :       Any    ):
								UpperCamelCase__		:     List[Any]								=self.model_tester.prepare_config_and_inputs()
								for type in ["absolute", "relative_key", "relative_key_query"]:
													UpperCamelCase__		:     List[Any]								=type
													self.model_tester.create_and_check_model(*lowercase_    )






			def  _lowerCAmelCase      (				self  :       Tuple    ):
								UpperCamelCase__		:     Dict								=self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_masked_lm(*lowercase_    )






			def  _lowerCAmelCase      (				self  :       Dict    ):
								UpperCamelCase__		:     str								=self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_sequence_classification(*lowercase_    )






			def  _lowerCAmelCase      (				self  :       Dict    ):
								UpperCamelCase__		:     Optional[int]								=self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_token_classification(*lowercase_    )






			def  _lowerCAmelCase      (				self  :       Tuple    ):
								UpperCamelCase__		:     Tuple								=self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_question_answering(*lowercase_    )






			@unittest.skip('''Will be fixed soon by reducing the size of the model used for common tests.'''    )
			def  _lowerCAmelCase      (				self  :       str    ):
								pass



def 							_lowerCAmelCase   (    ):


					'''simple docstring'''






					UpperCamelCase__		:     List[Any]								=torch.tensor([[101,1_019,1_014,1_016,1_037,12_849,4_747,1_004,14_246,2_278,5_439,4_524,5_002,2_930,2_193,2_930,4_341,3_208,1_005,1_055,2_171,2_848,11_300,3_531,102],[101,4_070,4_034,7_020,1_024,3_058,1_015,1_013,2_861,1_013,6_070,19_274,2_772,6_205,27_814,16_147,16_147,4_343,2_047,10_283,10_969,14_389,1_012,2_338,102]]   ,				device=UpperCAmelCase					)  # noqa: E231
					UpperCamelCase__		:     List[str]								=torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],]   ,				device=UpperCAmelCase					)  # noqa: E231
					UpperCamelCase__		:     Union[str, Any]								=torch.tensor([[[0,0,0,0],[423,237,440,251],[427,272,441,287],[419,115,437,129],[961,885,992,912],[256,38,330,58],[256,38,330,58],[336,42,353,57],[360,39,401,56],[360,39,401,56],[411,39,471,59],[479,41,528,59],[533,39,630,60],[67,113,134,131],[141,115,209,132],[68,149,133,166],[141,149,187,164],[195,148,287,165],[195,148,287,165],[195,148,287,165],[295,148,349,165],[441,149,492,166],[497,149,546,164],[64,201,125,218],[1_000,1_000,1_000,1_000]],[[0,0,0,0],[662,150,754,166],[665,199,742,211],[519,213,554,228],[519,213,554,228],[134,433,187,454],[130,467,204,480],[130,467,204,480],[130,467,204,480],[130,467,204,480],[130,467,204,480],[314,469,376,482],[504,684,582,706],[941,825,973,900],[941,825,973,900],[941,825,973,900],[941,825,973,900],[610,749,652,765],[130,659,168,672],[176,657,237,672],[238,657,312,672],[443,653,628,672],[443,653,628,672],[716,301,825,317],[1_000,1_000,1_000,1_000]]]   ,				device=UpperCAmelCase					)  # noqa: E231
					UpperCamelCase__		:     Union[str, Any]								=torch.tensor([[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]]   ,				device=UpperCAmelCase					)  # noqa: E231
					# these are sequence labels (i.e. at the token level)
					UpperCamelCase__		:     str								=torch.tensor([[-100,10,10,10,9,1,-100,7,7,-100,7,7,4,2,5,2,8,8,-100,-100,5,0,3,2,-100],[-100,12,12,12,-100,12,10,-100,-100,-100,-100,10,12,9,-100,-100,-100,10,10,10,9,12,-100,10,-100]]   ,				device=UpperCAmelCase					)  # noqa: E231
					# fmt: on

					return input_ids, attention_mask, bbox, token_type_ids, labels







@require_torch
class      __a (   unittest.TestCase     ):

			"""simple docstring"""

			@slow
			def  _lowerCAmelCase      (				self  :       Optional[int]    ):
								UpperCamelCase__		:     str								=LayoutLMModel.from_pretrained('''microsoft/layoutlm-base-uncased'''    ).to(lowercase_    )

								UpperCamelCase__	,      UpperCamelCase__	,      UpperCamelCase__	,      UpperCamelCase__	,      UpperCamelCase__		:     Tuple								=prepare_layoutlm_batch_inputs()

								# forward pass
								UpperCamelCase__		:     Dict								=model(input_ids=lowercase_    ,	bbox=lowercase_    ,	attention_mask=lowercase_    ,	token_type_ids=lowercase_    )

								# test the sequence output on [0, :3, :3]
								UpperCamelCase__		:     str								=torch.tensor(
								    [[0.1_7_8_5, -0.1_9_4_7, -0.0_4_2_5], [-0.3_2_5_4, -0.2_8_0_7, 0.2_5_5_3], [-0.5_3_9_1, -0.3_3_2_2, 0.3_3_6_4]]    ,	device=lowercase_    ,	)

								self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]    ,	lowercase_    ,	atol=1e-3    )    )

								# test the pooled output on [1, :3]
								UpperCamelCase__		:     str								=torch.tensor([-0.6_5_8_0, -0.0_2_1_4, 0.8_5_5_2]    ,	device=lowercase_    )

								self.assertTrue(torch.allclose(outputs.pooler_output[1, :3]    ,	lowercase_    ,	atol=1e-3    )    )






			@slow
			def  _lowerCAmelCase      (				self  :       int    ):
								# initialize model with randomly initialized sequence classification head
								UpperCamelCase__		:     int								=LayoutLMForSequenceClassification.from_pretrained('''microsoft/layoutlm-base-uncased'''    ,	num_labels=2    ).to(
								    lowercase_    )

								UpperCamelCase__	,      UpperCamelCase__	,      UpperCamelCase__	,      UpperCamelCase__	,      UpperCamelCase__		:     Any								=prepare_layoutlm_batch_inputs()

								# forward pass
								UpperCamelCase__		:     str								=model(
								    input_ids=lowercase_    ,	bbox=lowercase_    ,	attention_mask=lowercase_    ,	token_type_ids=lowercase_    ,	labels=torch.tensor([1, 1]    ,	device=lowercase_    )    ,	)

								# test whether we get a loss as a scalar
								UpperCamelCase__		:     Optional[int]								=outputs.loss
								UpperCamelCase__		:     Tuple								=torch.Size([]    )
								self.assertEqual(loss.shape    ,	lowercase_    )

								# test the shape of the logits
								UpperCamelCase__		:     Any								=outputs.logits
								UpperCamelCase__		:     Optional[Any]								=torch.Size((2, 2)    )
								self.assertEqual(logits.shape    ,	lowercase_    )






			@slow
			def  _lowerCAmelCase      (				self  :       List[Any]    ):
								# initialize model with randomly initialized token classification head
								UpperCamelCase__		:     List[str]								=LayoutLMForTokenClassification.from_pretrained('''microsoft/layoutlm-base-uncased'''    ,	num_labels=13    ).to(
								    lowercase_    )

								UpperCamelCase__	,      UpperCamelCase__	,      UpperCamelCase__	,      UpperCamelCase__	,      UpperCamelCase__		:     Any								=prepare_layoutlm_batch_inputs()

								# forward pass
								UpperCamelCase__		:     List[Any]								=model(
								    input_ids=lowercase_    ,	bbox=lowercase_    ,	attention_mask=lowercase_    ,	token_type_ids=lowercase_    ,	labels=lowercase_    )

								# test the loss calculation to be around 2.65
								# expected_loss = torch.tensor(2.65, device=torch_device)

								# The loss is currently somewhat random and can vary between 0.1-0.3 atol.
								# self.assertTrue(torch.allclose(outputs.loss, expected_loss, atol=0.1))

								# test the shape of the logits
								UpperCamelCase__		:     Union[str, Any]								=outputs.logits
								UpperCamelCase__		:     List[str]								=torch.Size((2, 25, 13)    )
								self.assertEqual(logits.shape    ,	lowercase_    )






			@slow
			def  _lowerCAmelCase      (				self  :       Tuple    ):
								# initialize model with randomly initialized token classification head
								UpperCamelCase__		:     Tuple								=LayoutLMForQuestionAnswering.from_pretrained('''microsoft/layoutlm-base-uncased'''    ).to(lowercase_    )

								UpperCamelCase__	,      UpperCamelCase__	,      UpperCamelCase__	,      UpperCamelCase__	,      UpperCamelCase__		:     Any								=prepare_layoutlm_batch_inputs()

								# forward pass
								UpperCamelCase__		:     Optional[int]								=model(input_ids=lowercase_    ,	bbox=lowercase_    ,	attention_mask=lowercase_    ,	token_type_ids=lowercase_    )

								# test the shape of the logits
								UpperCamelCase__		:     Optional[Any]								=torch.Size((2, 25)    )
								self.assertEqual(outputs.start_logits.shape    ,	lowercase_    )
								self.assertEqual(outputs.end_logits.shape    ,	lowercase_    )





"""simple docstring"""





import argparse
import json
import re
import string
import sys

import numpy as np


_SCREAMING_SNAKE_CASE							:	str										=			0.5



def 							_lowerCAmelCase   (    UpperCAmelCase  :				str   ,				UpperCAmelCase  :				Optional[int]					):


					'''simple docstring'''






					UpperCamelCase__		:     Optional[Any]								=['''.''', ''',''', ''';''', ''':''']

					for token in remove_tokens:
										UpperCamelCase__		:     Optional[int]								=ground_truth.replace(UpperCAmelCase   ,				''''''					)
										UpperCamelCase__		:     List[Any]								=prediction.replace(UpperCAmelCase   ,				''''''					)

					UpperCamelCase__	,      UpperCamelCase__		:     List[Any]								=ground_truth.lower(), prediction.lower()
					UpperCamelCase__	,      UpperCamelCase__		:     Dict								=ground_truth.replace('''/'''   ,				''' '''					), prediction.replace('''/'''   ,				''' '''					)
					UpperCamelCase__	,      UpperCamelCase__		:     Any								=set(ground_truth.split(''' '''					)					), set(prediction.split(''' '''					)					)

					UpperCamelCase__		:     Optional[Any]								=ground_truth.intersection(UpperCAmelCase					)
					UpperCamelCase__		:     Optional[Any]								=ground_truth.union(UpperCAmelCase					)
					UpperCamelCase__		:     str								=len(UpperCAmelCase					) / len(UpperCAmelCase					)
					return jaccard



def 							_lowerCAmelCase   (    UpperCAmelCase  :				Union[str, Any]					):


					'''simple docstring'''






					def remove_articles(UpperCAmelCase  :				int					):
										return re.sub(r'''\b(a|an|the)\b'''   ,				''' '''   ,				UpperCAmelCase					)

					def white_space_fix(UpperCAmelCase  :				Union[str, Any]					):
										return " ".join(text.split()					)

					def remove_punc(UpperCAmelCase  :				Optional[int]					):
										UpperCamelCase__		:     Dict								=set(string.punctuation					)
										return "".join(ch for ch in text if ch not in exclude					)

					def lower(UpperCAmelCase  :				Optional[Any]					):
										return text.lower()

					return white_space_fix(remove_articles(remove_punc(lower(UpperCAmelCase					)					)					)					)



def 							_lowerCAmelCase   (    UpperCAmelCase  :				int   ,				UpperCAmelCase  :				str   ,				UpperCAmelCase  :				Any					):


					'''simple docstring'''






					UpperCamelCase__	,      UpperCamelCase__	,      UpperCamelCase__		:     Optional[Any]								=0, 0, 0

					UpperCamelCase__		:     int								='''Parties''' in qa_id

					# first check if ground truth is empty
					if len(UpperCAmelCase					) == 0:
										if len(UpperCAmelCase					) > 0:
															fp += len(UpperCAmelCase					)  # false positive for each one
					else:
										for ground_truth in ground_truths:
															assert len(UpperCAmelCase					) > 0
															# check if there is a match
															UpperCamelCase__		:     Union[str, Any]								=False
															for pred in predictions:
																				if substr_ok:
																									UpperCamelCase__		:     List[str]								=get_jaccard(UpperCAmelCase   ,				UpperCAmelCase					) >= IOU_THRESH or ground_truth in pred
																				else:
																									UpperCamelCase__		:     Union[str, Any]								=get_jaccard(UpperCAmelCase   ,				UpperCAmelCase					) >= IOU_THRESH
																				if is_match:
																									UpperCamelCase__		:     Dict								=True

															if match_found:
																				tp += 1
															else:
																				fn += 1

        # now also get any fps by looping through preds
										for pred in predictions:
															# Check if there's a match. if so, don't count (don't want to double count based on the above)
															# but if there's no match, then this is a false positive.
															# (Note: we get the true positives in the above loop instead of this loop so that we don't double count
															# multiple predictions that are matched with the same answer.)
															UpperCamelCase__		:     Any								=False
															for ground_truth in ground_truths:
																				assert len(UpperCAmelCase					) > 0
																				if substr_ok:
																									UpperCamelCase__		:     Union[str, Any]								=get_jaccard(UpperCAmelCase   ,				UpperCAmelCase					) >= IOU_THRESH or ground_truth in pred
																				else:
																									UpperCamelCase__		:     Dict								=get_jaccard(UpperCAmelCase   ,				UpperCAmelCase					) >= IOU_THRESH
																				if is_match:
																									UpperCamelCase__		:     str								=True

															if not match_found:
																				fp += 1

					UpperCamelCase__		:     Union[str, Any]								=tp / (tp + fp) if tp + fp > 0 else np.nan
					UpperCamelCase__		:     Optional[int]								=tp / (tp + fn) if tp + fn > 0 else np.nan

					return precision, recall



def 							_lowerCAmelCase   (    UpperCAmelCase  :				List[Any]					):


					'''simple docstring'''






					UpperCamelCase__		:     Optional[Any]								=precisions[::-1]
					for i in range(1   ,				len(UpperCAmelCase					)					):
										UpperCamelCase__		:     int								=max(precision_best[i - 1]   ,				precision_best[i]					)
					UpperCamelCase__		:     str								=precision_best[::-1]
					return precisions



def 							_lowerCAmelCase   (    UpperCAmelCase  :				List[str]   ,				UpperCAmelCase  :				Any					):


					'''simple docstring'''






					UpperCamelCase__		:     int								=process_precisions(UpperCAmelCase					)
					UpperCamelCase__		:     str								=np.trapz(UpperCAmelCase   ,				UpperCAmelCase					)
					if np.isnan(UpperCAmelCase					):
										return 0
					return aupr



def 							_lowerCAmelCase   (    UpperCAmelCase  :				Dict   ,				UpperCAmelCase  :				int   ,				UpperCAmelCase  :				int					):


					'''simple docstring'''






					UpperCamelCase__		:     List[Any]								=process_precisions(UpperCAmelCase					)
					UpperCamelCase__		:     Optional[int]								=0
					for prec, recall in zip(UpperCAmelCase   ,				UpperCAmelCase					):
										if recall >= recall_thresh:
															UpperCamelCase__		:     Any								=prec
															break
					return prec_at_recall



def 							_lowerCAmelCase   (    UpperCAmelCase  :				Any   ,				UpperCAmelCase  :				Dict					):


					'''simple docstring'''






					return normalize_answer(UpperCAmelCase					) == normalize_answer(UpperCAmelCase					)



def 							_lowerCAmelCase   (    UpperCAmelCase  :				int   ,				UpperCAmelCase  :				Dict   ,				UpperCAmelCase  :				int					):


					'''simple docstring'''






					UpperCamelCase__		:     List[str]								=0
					for pred in predictions:
										for ground_truth in ground_truths:
															UpperCamelCase__		:     str								=metric_fn(UpperCAmelCase   ,				UpperCAmelCase					)
															if score == 1:  # break the loop when one prediction matches the ground truth
																				break
										if score == 1:
															break
					return score



def 							_lowerCAmelCase   (    UpperCAmelCase  :				Optional[int]   ,				UpperCAmelCase  :				str					):


					'''simple docstring'''






					UpperCamelCase__		:     Any								=0
					UpperCamelCase__		:     List[Any]								=[]
					UpperCamelCase__		:     Optional[Any]								=[]
					for article in dataset:
										for paragraph in article["paragraphs"]:
															for qa in paragraph["qas"]:
																				total += 1
																				if qa["id"] not in predictions:
																									UpperCamelCase__		:     Union[str, Any]								='''Unanswered question ''' + qa['''id'''] + ''' will receive score 0.'''
																									print(UpperCAmelCase   ,				file=sys.stderr					)
																									continue
																				UpperCamelCase__		:     Union[str, Any]								=[x['''text'''] for x in qa['''answers''']]
																				UpperCamelCase__		:     Any								=predictions[qa['''id''']]
																				UpperCamelCase__	,      UpperCamelCase__		:     int								=compute_precision_recall(UpperCAmelCase   ,				UpperCAmelCase   ,				qa['''id''']					)

																				precisions.append(UpperCAmelCase					)
																				recalls.append(UpperCAmelCase					)

																				if precision == 0 and recall == 0:
																									fa += 0
																				else:
																									fa += 2 * (precision * recall) / (precision + recall)

																				exact_match += metric_max_over_ground_truths(UpperCAmelCase   ,				UpperCAmelCase   ,				UpperCAmelCase					)

					UpperCamelCase__		:     Union[str, Any]								=[x for _, x in sorted(zip(UpperCAmelCase   ,				UpperCAmelCase					)					)]
					recalls.sort()

					UpperCamelCase__		:     Optional[int]								=100.0 * fa / total
					UpperCamelCase__		:     List[Any]								=100.0 * exact_match / total
					UpperCamelCase__		:     Dict								=get_aupr(UpperCAmelCase   ,				UpperCAmelCase					)

					UpperCamelCase__		:     Optional[Any]								=get_prec_at_recall(UpperCAmelCase   ,				UpperCAmelCase   ,				recall_thresh=0.9					)
					UpperCamelCase__		:     Dict								=get_prec_at_recall(UpperCAmelCase   ,				UpperCAmelCase   ,				recall_thresh=0.8					)

					return {
					    "exact_match": exact_match,
					    "f1": fa,
					    "aupr": aupr,
					    "prec_at_80_recall": prec_at_aa_recall,
					    "prec_at_90_recall": prec_at_aa_recall,
					}


if __name__ == "__main__":
				_SCREAMING_SNAKE_CASE							:	str										=			argparse.ArgumentParser(description="""Evaluation for CUAD""")
				parser.add_argument("""dataset_file""", help="""Dataset file""")
				parser.add_argument("""prediction_file""", help="""Prediction File""")
				_SCREAMING_SNAKE_CASE							:	Dict										=			parser.parse_args()
				with open(args.dataset_file) as dataset_file:
								_SCREAMING_SNAKE_CASE							:	str										=			json.load(dataset_file)
								_SCREAMING_SNAKE_CASE							:	Tuple										=			dataset_json["""data"""]
				with open(args.prediction_file) as prediction_file:
								_SCREAMING_SNAKE_CASE							:	Dict										=			json.load(prediction_file)
				print(json.dumps(evaluate(dataset, predictions)))


