





"""simple docstring"""




import collections
import inspect
import unittest

from transformers import DonutSwinConfig
from transformers.testing_utils import require_torch, slow, torch_device
from transformers.utils import is_torch_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch
						from torch import nn

						from transformers import DonutSwinModel
						from transformers.models.donut.modeling_donut_swin import DONUT_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST




class 					_a		:






				def __init__(					self						:       Dict,					lowerCAmelCase__						:       Optional[int],					lowerCAmelCase__						:       str=1_3,					lowerCAmelCase__						:       str=3_2,					lowerCAmelCase__						:       List[str]=2,					lowerCAmelCase__						:       Dict=3,					lowerCAmelCase__						:       Dict=1_6,					lowerCAmelCase__						:       Tuple=[1, 2, 1],					lowerCAmelCase__						:       Optional[int]=[2, 2, 4],					lowerCAmelCase__						:       Any=2,					lowerCAmelCase__						:       Optional[Any]=2.0,					lowerCAmelCase__						:       List[str]=True,					lowerCAmelCase__						:       int=0.0,					lowerCAmelCase__						:       Any=0.0,					lowerCAmelCase__						:       Optional[Any]=0.1,					lowerCAmelCase__						:       Any="gelu",					lowerCAmelCase__						:       str=False,					lowerCAmelCase__						:       Optional[int]=True,					lowerCAmelCase__						:       Optional[int]=0.02,					lowerCAmelCase__						:       Dict=1e-5,					lowerCAmelCase__						:       Any=True,					lowerCAmelCase__						:       Union[str, Any]=None,					lowerCAmelCase__						:       List[Any]=True,					lowerCAmelCase__						:       Optional[Any]=1_0,					lowerCAmelCase__						:       Any=8,					)      -> List[Any]:




										'''simple docstring'''



										_UpperCamelCase     :   str      =       parent
										_UpperCamelCase     :   List[Any]      =       batch_size
										_UpperCamelCase     :   Optional[int]      =       image_size
										_UpperCamelCase     :   Tuple      =       patch_size
										_UpperCamelCase     :   Union[str, Any]      =       num_channels
										_UpperCamelCase     :   List[str]      =       embed_dim
										_UpperCamelCase     :   List[Any]      =       depths
										_UpperCamelCase     :   Tuple      =       num_heads
										_UpperCamelCase     :   Union[str, Any]      =       window_size
										_UpperCamelCase     :   Union[str, Any]      =       mlp_ratio
										_UpperCamelCase     :   List[Any]      =       qkv_bias
										_UpperCamelCase     :   Dict      =       hidden_dropout_prob
										_UpperCamelCase     :   List[str]      =       attention_probs_dropout_prob
										_UpperCamelCase     :   Dict      =       drop_path_rate
										_UpperCamelCase     :   Any      =       hidden_act
										_UpperCamelCase     :   Union[str, Any]      =       use_absolute_embeddings
										_UpperCamelCase     :   Dict      =       patch_norm
										_UpperCamelCase     :   int      =       layer_norm_eps
										_UpperCamelCase     :   List[Any]      =       initializer_range
										_UpperCamelCase     :   str      =       is_training
										_UpperCamelCase     :   Tuple      =       scope
										_UpperCamelCase     :   Optional[int]      =       use_labels
										_UpperCamelCase     :   Union[str, Any]      =       type_sequence_label_size
										_UpperCamelCase     :   List[str]      =       encoder_stride






				def 							snake_case      (					self						:       int				)      -> Union[str, Any]:




										'''simple docstring'''



										_UpperCamelCase     :   List[str]      =       floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]				)

										_UpperCamelCase     :   int      =       None
										if self.use_labels:
																_UpperCamelCase     :   Optional[int]      =       ids_tensor([self.batch_size],					self.type_sequence_label_size				)

										_UpperCamelCase     :   str      =       self.get_config()

										return config, pixel_values, labels






				def 							snake_case      (					self						:       List[str]				)      -> List[Any]:




										'''simple docstring'''



										return DonutSwinConfig(
										    image_size=self.image_size,					patch_size=self.patch_size,					num_channels=self.num_channels,					embed_dim=self.embed_dim,					depths=self.depths,					num_heads=self.num_heads,					window_size=self.window_size,					mlp_ratio=self.mlp_ratio,					qkv_bias=self.qkv_bias,					hidden_dropout_prob=self.hidden_dropout_prob,					attention_probs_dropout_prob=self.attention_probs_dropout_prob,					drop_path_rate=self.drop_path_rate,					hidden_act=self.hidden_act,					use_absolute_embeddings=self.use_absolute_embeddings,					path_norm=self.patch_norm,					layer_norm_eps=self.layer_norm_eps,					initializer_range=self.initializer_range,					encoder_stride=self.encoder_stride,					)






				def 							snake_case      (					self						:       List[str],					lowerCAmelCase__						:       Dict,					lowerCAmelCase__						:       List[str],					lowerCAmelCase__						:       Optional[Any]				)      -> Union[str, Any]:




										'''simple docstring'''



										_UpperCamelCase     :   Tuple      =       DonutSwinModel(config=lowerCAmelCase__				)
										model.to(lowerCAmelCase__				)
										model.eval()
										_UpperCamelCase     :   Optional[int]      =       model(lowerCAmelCase__				)

										_UpperCamelCase     :   List[Any]      =       ((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths				) - 1))
										_UpperCamelCase     :   Union[str, Any]      =       int(config.embed_dim * 2 ** (len(config.depths				) - 1)				)

										self.parent.assertEqual(result.last_hidden_state.shape,					(self.batch_size, expected_seq_len, expected_dim)				)






				def 							snake_case      (					self						:       Union[str, Any]				)      -> Dict:




										'''simple docstring'''



										_UpperCamelCase     :   str      =       self.prepare_config_and_inputs()
										(
										    (
										    _UpperCamelCase
										)  ,				(
										    _UpperCamelCase
										)  ,				(
										    _UpperCamelCase
										)  ,				
										)     :   List[str]      =       config_and_inputs
										_UpperCamelCase     :   Tuple      =       {'''pixel_values''': pixel_values}
										return config, inputs_dict




@require_torch
class 					_a		( _lowerCAmelCase      ,							_lowerCAmelCase      ,							unittest.TestCase							):
				UpperCamelCase								=       (DonutSwinModel,) if is_torch_available() else ()
				UpperCamelCase								=       {'''feature-extraction''': DonutSwinModel} if is_torch_available() else {}
				UpperCamelCase								=       True

				UpperCamelCase								=       False
				UpperCamelCase								=       False
				UpperCamelCase								=       False






				def 							snake_case      (					self						:       List[str]				)      -> Dict:




										'''simple docstring'''



										_UpperCamelCase     :   Union[str, Any]      =       DonutSwinModelTester(self				)
										_UpperCamelCase     :   Union[str, Any]      =       ConfigTester(self,					config_class=lowerCAmelCase__,					embed_dim=3_7				)






				def 							snake_case      (					self						:       int				)      -> List[Any]:




										'''simple docstring'''



										self.create_and_test_config_common_properties()
										self.config_tester.create_and_test_config_to_json_string()
										self.config_tester.create_and_test_config_to_json_file()
										self.config_tester.create_and_test_config_from_and_save_pretrained()
										self.config_tester.create_and_test_config_with_num_labels()
										self.config_tester.check_config_can_be_init_without_params()
										self.config_tester.check_config_arguments_init()






				def 							snake_case      (					self						:       Any				)      -> int:




										'''simple docstring'''



										return






				def 							snake_case      (					self						:       Tuple				)      -> Any:




										'''simple docstring'''



										_UpperCamelCase     :   int      =       self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*lowerCAmelCase__				)






				def 							snake_case      (					self						:       List[Any]				)      -> Dict:




										'''simple docstring'''



										pass






				def 							snake_case      (					self						:       Dict				)      -> Union[str, Any]:




										'''simple docstring'''



										_UpperCamelCase  ,				_UpperCamelCase     :   Optional[Any]      =       self.model_tester.prepare_config_and_inputs_for_common()

										for model_class in self.all_model_classes:
																_UpperCamelCase     :   Any      =       model_class(lowerCAmelCase__				)
																self.assertIsInstance(model.get_input_embeddings(),					(nn.Module)				)
																_UpperCamelCase     :   Union[str, Any]      =       model.get_output_embeddings()
																self.assertTrue(x is None or isinstance(lowerCAmelCase__,					nn.Linear				)				)






				def 							snake_case      (					self						:       List[str]				)      -> Dict:




										'''simple docstring'''



										_UpperCamelCase  ,				_UpperCamelCase     :   Union[str, Any]      =       self.model_tester.prepare_config_and_inputs_for_common()

										for model_class in self.all_model_classes:
																_UpperCamelCase     :   Any      =       model_class(lowerCAmelCase__				)
																_UpperCamelCase     :   List[Any]      =       inspect.signature(model.forward				)
																# signature.parameters is an OrderedDict => so arg_names order is deterministic
																_UpperCamelCase     :   str      =       [*signature.parameters.keys()]

																_UpperCamelCase     :   str      =       ['''pixel_values''']
																self.assertListEqual(arg_names[:1],					lowerCAmelCase__				)






				def 							snake_case      (					self						:       List[str]				)      -> int:




										'''simple docstring'''



										_UpperCamelCase  ,				_UpperCamelCase     :   Union[str, Any]      =       self.model_tester.prepare_config_and_inputs_for_common()
										_UpperCamelCase     :   Optional[Any]      =       True

										for model_class in self.all_model_classes:
																_UpperCamelCase     :   str      =       True
																_UpperCamelCase     :   int      =       False
																_UpperCamelCase     :   int      =       True
																_UpperCamelCase     :   Dict      =       model_class(lowerCAmelCase__				)
																model.to(lowerCAmelCase__				)
																model.eval()
																with torch.no_grad():
																						_UpperCamelCase     :   Dict      =       model(**self._prepare_for_class(lowerCAmelCase__,					lowerCAmelCase__				)				)
																_UpperCamelCase     :   Union[str, Any]      =       outputs.attentions
																_UpperCamelCase     :   Any      =       len(self.model_tester.depths				)
																self.assertEqual(len(lowerCAmelCase__				),					lowerCAmelCase__				)

																# check that output_attentions also work using config
																del inputs_dict["output_attentions"]
																_UpperCamelCase     :   Optional[Any]      =       True
																_UpperCamelCase     :   str      =       config.window_size**2
																_UpperCamelCase     :   Dict      =       model_class(lowerCAmelCase__				)
																model.to(lowerCAmelCase__				)
																model.eval()
																with torch.no_grad():
																						_UpperCamelCase     :   Optional[Any]      =       model(**self._prepare_for_class(lowerCAmelCase__,					lowerCAmelCase__				)				)
																_UpperCamelCase     :   str      =       outputs.attentions
																self.assertEqual(len(lowerCAmelCase__				),					lowerCAmelCase__				)

																self.assertListEqual(
																    list(attentions[0].shape[-3:]				),					[self.model_tester.num_heads[0], window_size_squared, window_size_squared],					)
																_UpperCamelCase     :   List[Any]      =       len(lowerCAmelCase__				)

																# Check attention is always last and order is fine
																_UpperCamelCase     :   List[str]      =       True
																_UpperCamelCase     :   Dict      =       True
																_UpperCamelCase     :   Any      =       model_class(lowerCAmelCase__				)
																model.to(lowerCAmelCase__				)
																model.eval()
																with torch.no_grad():
																						_UpperCamelCase     :   int      =       model(**self._prepare_for_class(lowerCAmelCase__,					lowerCAmelCase__				)				)

																if hasattr(self.model_tester,					'''num_hidden_states_types'''				):
																						_UpperCamelCase     :   int      =       self.model_tester.num_hidden_states_types
																else:
																						# also another +1 for reshaped_hidden_states
																						_UpperCamelCase     :   Any      =       2
																self.assertEqual(out_len + added_hidden_states,					len(lowerCAmelCase__				)				)

																_UpperCamelCase     :   List[str]      =       outputs.attentions

																self.assertEqual(len(lowerCAmelCase__				),					lowerCAmelCase__				)

																self.assertListEqual(
																    list(self_attentions[0].shape[-3:]				),					[self.model_tester.num_heads[0], window_size_squared, window_size_squared],					)






				def 							snake_case      (					self						:       List[str],					lowerCAmelCase__						:       Optional[Any],					lowerCAmelCase__						:       List[Any],					lowerCAmelCase__						:       Optional[int],					lowerCAmelCase__						:       Optional[Any]				)      -> Tuple:




										'''simple docstring'''



										_UpperCamelCase     :   Union[str, Any]      =       model_class(lowerCAmelCase__				)
										model.to(lowerCAmelCase__				)
										model.eval()

										with torch.no_grad():
																_UpperCamelCase     :   Optional[int]      =       model(**self._prepare_for_class(lowerCAmelCase__,					lowerCAmelCase__				)				)

										_UpperCamelCase     :   Any      =       outputs.hidden_states

										_UpperCamelCase     :   Tuple      =       getattr(
										    self.model_tester,					'''expected_num_hidden_layers''',					len(self.model_tester.depths				) + 1				)
										self.assertEqual(len(lowerCAmelCase__				),					lowerCAmelCase__				)

										# DonutSwin has a different seq_length
										_UpperCamelCase     :   List[Any]      =       (
										    config.patch_size
										    if isinstance(config.patch_size,					collections.abc.Iterable				)
										    else (config.patch_size, config.patch_size)
										)

										_UpperCamelCase     :   List[Any]      =       (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

										self.assertListEqual(
										    list(hidden_states[0].shape[-2:]				),					[num_patches, self.model_tester.embed_dim],					)

										_UpperCamelCase     :   Union[str, Any]      =       outputs.reshaped_hidden_states
										self.assertEqual(len(lowerCAmelCase__				),					lowerCAmelCase__				)

										_UpperCamelCase  ,				_UpperCamelCase  ,				_UpperCamelCase  ,				_UpperCamelCase     :   Union[str, Any]      =       reshaped_hidden_states[0].shape
										_UpperCamelCase     :   Optional[Any]      =       (
										    reshaped_hidden_states[0].view(lowerCAmelCase__,					lowerCAmelCase__,					height * width				).permute(0,					2,					1				)
										)
										self.assertListEqual(
										    list(reshaped_hidden_states.shape[-2:]				),					[num_patches, self.model_tester.embed_dim],					)






				def 							snake_case      (					self						:       List[Any]				)      -> List[Any]:




										'''simple docstring'''



										_UpperCamelCase  ,				_UpperCamelCase     :   Optional[Any]      =       self.model_tester.prepare_config_and_inputs_for_common()

										_UpperCamelCase     :   int      =       (
										    self.model_tester.image_size
										    if isinstance(self.model_tester.image_size,					collections.abc.Iterable				)
										    else (self.model_tester.image_size, self.model_tester.image_size)
										)

										for model_class in self.all_model_classes:
																_UpperCamelCase     :   int      =       True
																self.check_hidden_states_output(lowerCAmelCase__,					lowerCAmelCase__,					lowerCAmelCase__,					lowerCAmelCase__				)

																# check that output_hidden_states also work using config
																del inputs_dict["output_hidden_states"]
																_UpperCamelCase     :   Any      =       True

																self.check_hidden_states_output(lowerCAmelCase__,					lowerCAmelCase__,					lowerCAmelCase__,					lowerCAmelCase__				)






				def 							snake_case      (					self						:       Dict				)      -> Tuple:




										'''simple docstring'''



										_UpperCamelCase  ,				_UpperCamelCase     :   int      =       self.model_tester.prepare_config_and_inputs_for_common()
										_UpperCamelCase     :   Any      =       3

										_UpperCamelCase     :   Tuple      =       (
										    self.model_tester.image_size
										    if isinstance(self.model_tester.image_size,					collections.abc.Iterable				)
										    else (self.model_tester.image_size, self.model_tester.image_size)
										)
										_UpperCamelCase     :   Any      =       (
										    config.patch_size
										    if isinstance(config.patch_size,					collections.abc.Iterable				)
										    else (config.patch_size, config.patch_size)
										)

										_UpperCamelCase     :   Optional[int]      =       image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
										_UpperCamelCase     :   int      =       image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

										for model_class in self.all_model_classes:
																_UpperCamelCase     :   Union[str, Any]      =       True
																self.check_hidden_states_output(lowerCAmelCase__,					lowerCAmelCase__,					lowerCAmelCase__,					(padded_height, padded_width)				)

																# check that output_hidden_states also work using config
																del inputs_dict["output_hidden_states"]
																_UpperCamelCase     :   Dict      =       True
																self.check_hidden_states_output(lowerCAmelCase__,					lowerCAmelCase__,					lowerCAmelCase__,					(padded_height, padded_width)				)






				@slow
				def 							snake_case      (					self						:       Any				)      -> Tuple:




										'''simple docstring'''



										for model_name in DONUT_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																_UpperCamelCase     :   Union[str, Any]      =       DonutSwinModel.from_pretrained(lowerCAmelCase__				)
																self.assertIsNotNone(lowerCAmelCase__				)






				def 							snake_case      (					self						:       str				)      -> Optional[int]:




										'''simple docstring'''



										_UpperCamelCase  ,				_UpperCamelCase     :   Tuple      =       self.model_tester.prepare_config_and_inputs_for_common()

										_UpperCamelCase     :   str      =       _config_zero_init(lowerCAmelCase__				)
										for model_class in self.all_model_classes:
																_UpperCamelCase     :   Optional[Any]      =       model_class(config=lowerCAmelCase__				)
																for name, param in model.named_parameters():
																						if "embeddings" not in name and param.requires_grad:
																												self.assertIn(
																												    ((param.data.mean() * 1e9).round() / 1e9).item(),					[0.0, 1.0],					msg=f"""Parameter {name} of model {model_class} seems not properly initialized""",					)





"""simple docstring"""

import datasets

from .evaluate import (
    apply_no_ans_threshold,
    find_all_best_thresh,
    get_raw_scores,
    make_eval_dict,
    make_qid_to_has_ans,
    merge_eval,
)


UpperCamelCase      :							List[str]          =					"\\n@inproceedings{Rajpurkar2016SQuAD10,\n  title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text},\n  author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang},\n  booktitle={EMNLP},\n  year={2016}\n}\n"

UpperCamelCase      :							int          =					"\nThis metric wrap the official scoring script for version 2 of the Stanford Question\nAnswering Dataset (SQuAD).\n\nStanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by\ncrowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span,\nfrom the corresponding reading passage, or the question might be unanswerable.\n\nSQuAD2.0 combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions\nwritten adversarially by crowdworkers to look similar to answerable ones.\nTo do well on SQuAD2.0, systems must not only answer questions when possible, but also\ndetermine when no answer is supported by the paragraph and abstain from answering.\n"

UpperCamelCase      :							Optional[Any]          =					"\nComputes SQuAD v2 scores (F1 and EM).\nArgs:\n    predictions: List of triple for question-answers to score with the following elements:\n        - the question-answer 'id' field as given in the references (see below)\n        - the text of the answer\n        - the probability that the question has no answer\n    references: List of question-answers dictionaries with the following key-values:\n            - 'id': id of the question-answer pair (see above),\n            - 'answers': a list of Dict {'text': text of the answer as a string}\n    no_answer_threshold: float\n        Probability threshold to decide that a question has no answer.\nReturns:\n    'exact': Exact match (the normalized answer exactly match the gold answer)\n    'f1': The F-score of predicted tokens versus the gold answer\n    'total': Number of score considered\n    'HasAns_exact': Exact match (the normalized answer exactly match the gold answer)\n    'HasAns_f1': The F-score of predicted tokens versus the gold answer\n    'HasAns_total': Number of score considered\n    'NoAns_exact': Exact match (the normalized answer exactly match the gold answer)\n    'NoAns_f1': The F-score of predicted tokens versus the gold answer\n    'NoAns_total': Number of score considered\n    'best_exact': Best exact match (with varying threshold)\n    'best_exact_thresh': No-answer probability threshold associated to the best exact match\n    'best_f1': Best F1 (with varying threshold)\n    'best_f1_thresh': No-answer probability threshold associated to the best F1\nExamples:\n\n    >>> predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22', 'no_answer_probability': 0.}]\n    >>> references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}]\n    >>> squad_v2_metric = datasets.load_metric(\"squad_v2\")\n    >>> results = squad_v2_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact': 100.0, 'f1': 100.0, 'total': 1, 'HasAns_exact': 100.0, 'HasAns_f1': 100.0, 'HasAns_total': 1, 'best_exact': 100.0, 'best_exact_thresh': 0.0, 'best_f1': 100.0, 'best_f1_thresh': 0.0}\n"

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION     ,    _KWARGS_DESCRIPTION     )
class __lowerCAmelCase						( datasets.Metric     ):





     def 		UpperCAmelCase   (      self ):

      '''simple docstring'''




      return datasets.MetricInfo(
          description=_DESCRIPTION      ,	citation=_CITATION      ,	inputs_description=_KWARGS_DESCRIPTION      ,	features=datasets.Features(
              {
                  'predictions': {
                      'id': datasets.Value('string' ),
                      'prediction_text': datasets.Value('string' ),
                      'no_answer_probability': datasets.Value('float32' ),
                  },
                  'references': {
                      'id': datasets.Value('string' ),
                      'answers': datasets.features.Sequence(
                          {'text': datasets.Value('string' ), 'answer_start': datasets.Value('int32' )} ),
                  },
              } )      ,	codebase_urls=['https://rajpurkar.github.io/SQuAD-explorer/']      ,	reference_urls=['https://rajpurkar.github.io/SQuAD-explorer/']      ,	)







     def 		UpperCAmelCase   (      self      ,	__UpperCAmelCase      ,	__UpperCAmelCase      ,	__UpperCAmelCase=1.0 ):

      '''simple docstring'''




      __UpperCamelCase							    =			{p['id']: p['no_answer_probability'] for p in predictions}
      __UpperCamelCase							    =			[{'paragraphs': [{'qas': references}]}]
      __UpperCamelCase							    =			{p['id']: p['prediction_text'] for p in predictions}

      __UpperCamelCase							    =			make_qid_to_has_ans(__UpperCAmelCase )  # maps qid to True/False
      __UpperCamelCase							    =			[k for k, v in qid_to_has_ans.items() if v]
      __UpperCamelCase							    =			[k for k, v in qid_to_has_ans.items() if not v]

      __UpperCamelCase    ,      __UpperCamelCase							    =			get_raw_scores(__UpperCAmelCase      ,	__UpperCAmelCase )
      __UpperCamelCase							    =			apply_no_ans_threshold(__UpperCAmelCase      ,	__UpperCAmelCase      ,	__UpperCAmelCase      ,	__UpperCAmelCase )
      __UpperCamelCase							    =			apply_no_ans_threshold(__UpperCAmelCase      ,	__UpperCAmelCase      ,	__UpperCAmelCase      ,	__UpperCAmelCase )
      __UpperCamelCase							    =			make_eval_dict(__UpperCAmelCase      ,	__UpperCAmelCase )

      if has_ans_qids:
       __UpperCamelCase							    =			make_eval_dict(__UpperCAmelCase      ,	__UpperCAmelCase      ,	qid_list=__UpperCAmelCase )
       merge_eval(__UpperCAmelCase      ,	__UpperCAmelCase      ,	'HasAns' )
      if no_ans_qids:
       __UpperCamelCase							    =			make_eval_dict(__UpperCAmelCase      ,	__UpperCAmelCase      ,	qid_list=__UpperCAmelCase )
       merge_eval(__UpperCAmelCase      ,	__UpperCAmelCase      ,	'NoAns' )
      find_all_best_thresh(__UpperCAmelCase      ,	__UpperCAmelCase      ,	__UpperCAmelCase      ,	__UpperCAmelCase      ,	__UpperCAmelCase      ,	__UpperCAmelCase )
      return dict(__UpperCAmelCase )



