


from __future__ import annotations

import inspect
import unittest

import numpy as np

from transformers import SwinConfig
from transformers.testing_utils import require_tf, require_vision, slow, to_atuple
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import tensorflow as tf

 from transformers.models.swin.modeling_tf_swin import (
     TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
     TFSwinForImageClassification,
     TFSwinForMaskedImageModeling,
     TFSwinModel,
 )


if is_vision_available():
 from PIL import Image

 from transformers import AutoImageProcessor





class     __a     :
 def __init__(  self							,					lowerCAmelCase__							,					lowerCAmelCase__=13							,					lowerCAmelCase__=32							,					lowerCAmelCase__=2							,					lowerCAmelCase__=3							,					lowerCAmelCase__=16							,					lowerCAmelCase__=[1, 2, 1]							,					lowerCAmelCase__=[2, 2, 4]							,					lowerCAmelCase__=2							,					lowerCAmelCase__=2.0							,					lowerCAmelCase__=True							,					lowerCAmelCase__=0.0							,					lowerCAmelCase__=0.0							,					lowerCAmelCase__=0.1							,					lowerCAmelCase__="gelu"							,					lowerCAmelCase__=False							,					lowerCAmelCase__=True							,					lowerCAmelCase__=0.0_2							,					lowerCAmelCase__=1E-5							,					lowerCAmelCase__=True							,					lowerCAmelCase__=None							,					lowerCAmelCase__=True							,					lowerCAmelCase__=10							,					lowerCAmelCase__=8							,					)   ->							None:




        '''simple docstring'''
        lowercase__:  Optional[Any]         =  parent
        lowercase__:  Any         =  batch_size
        lowercase__:  List[str]         =  image_size
        lowercase__:  Optional[int]         =  patch_size
        lowercase__:  Optional[Any]         =  num_channels
        lowercase__:  List[str]         =  embed_dim
        lowercase__:  int         =  depths
        lowercase__:  Any         =  num_heads
        lowercase__:  str         =  window_size
        lowercase__:  List[str]         =  mlp_ratio
        lowercase__:  Optional[int]         =  qkv_bias
        lowercase__:  Any         =  hidden_dropout_prob
        lowercase__:  Any         =  attention_probs_dropout_prob
        lowercase__:  List[Any]         =  drop_path_rate
        lowercase__:  Tuple         =  hidden_act
        lowercase__:  int         =  use_absolute_embeddings
        lowercase__:  int         =  patch_norm
        lowercase__:  Any         =  layer_norm_eps
        lowercase__:  str         =  initializer_range
        lowercase__:  List[str]         =  is_training
        lowercase__:  Tuple         =  scope
        lowercase__:  List[Any]         =  use_labels
        lowercase__:  Tuple         =  type_sequence_label_size
        lowercase__:  str         =  encoder_stride
 def 						SCREAMING_SNAKE_CASE__  (  self					)   ->							int:




        '''simple docstring'''
        lowercase__:  int         =  floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]					)

        lowercase__:  Tuple         =  None
        if self.use_labels:
               lowercase__:  List[Any]         =  ids_tensor([self.batch_size]							,					self.type_sequence_label_size					)

        lowercase__:  Union[str, Any]         =  self.get_config()

        return config, pixel_values, labels
 def 						SCREAMING_SNAKE_CASE__  (  self					)   ->							int:




        '''simple docstring'''
        return SwinConfig(
            image_size=self.image_size							,					patch_size=self.patch_size							,					num_channels=self.num_channels							,					embed_dim=self.embed_dim							,					depths=self.depths							,					num_heads=self.num_heads							,					window_size=self.window_size							,					mlp_ratio=self.mlp_ratio							,					qkv_bias=self.qkv_bias							,					hidden_dropout_prob=self.hidden_dropout_prob							,					attention_probs_dropout_prob=self.attention_probs_dropout_prob							,					drop_path_rate=self.drop_path_rate							,					hidden_act=self.hidden_act							,					use_absolute_embeddings=self.use_absolute_embeddings							,					path_norm=self.patch_norm							,					layer_norm_eps=self.layer_norm_eps							,					initializer_range=self.initializer_range							,					encoder_stride=self.encoder_stride							,					)
 def 						SCREAMING_SNAKE_CASE__  (  self							,					lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__					)   ->							List[str]:




        '''simple docstring'''
        lowercase__:  Tuple         =  TFSwinModel(config=lowerCAmelCase__					)
        lowercase__:  List[Any]         =  model(lowerCAmelCase__					)

        lowercase__:  str         =  ((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths					) - 1))
        lowercase__:  Any         =  int(config.embed_dim * 2 ** (len(config.depths					) - 1)					)

        self.parent.assertEqual(result.last_hidden_state.shape							,					(self.batch_size, expected_seq_len, expected_dim)					)
 def 						SCREAMING_SNAKE_CASE__  (  self							,					lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__					)   ->							List[str]:




        '''simple docstring'''
        lowercase__:  List[str]         =  TFSwinForMaskedImageModeling(config=lowerCAmelCase__					)
        lowercase__:  Dict         =  model(lowerCAmelCase__					)
        self.parent.assertEqual(
            result.logits.shape							,					(self.batch_size, self.num_channels, self.image_size, self.image_size)					)

        # test greyscale images
        lowercase__:  int         =  1
        lowercase__:  Tuple         =  TFSwinForMaskedImageModeling(lowerCAmelCase__					)

        lowercase__:  str         =  floats_tensor([self.batch_size, 1, self.image_size, self.image_size]					)
        lowercase__:  Any         =  model(lowerCAmelCase__					)
        self.parent.assertEqual(result.logits.shape							,					(self.batch_size, 1, self.image_size, self.image_size)					)
 def 						SCREAMING_SNAKE_CASE__  (  self							,					lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__					)   ->							Optional[Any]:




        '''simple docstring'''
        lowercase__:  Any         =  self.type_sequence_label_size
        lowercase__:  List[Any]         =  TFSwinForImageClassification(lowerCAmelCase__					)
        lowercase__:  List[Any]         =  model(lowerCAmelCase__							,					labels=lowerCAmelCase__					)
        self.parent.assertEqual(result.logits.shape							,					(self.batch_size, self.type_sequence_label_size)					)

        # test greyscale images
        lowercase__:  List[Any]         =  1
        lowercase__:  Any         =  TFSwinForImageClassification(lowerCAmelCase__					)
        lowercase__:  Optional[int]         =  floats_tensor([self.batch_size, 1, self.image_size, self.image_size]					)
        lowercase__:  Dict         =  model(lowerCAmelCase__					)
        self.parent.assertEqual(result.logits.shape							,					(self.batch_size, self.type_sequence_label_size)					)
 def 						SCREAMING_SNAKE_CASE__  (  self					)   ->							Optional[int]:




        '''simple docstring'''
        lowercase__:  Tuple         =  self.prepare_config_and_inputs()
        lowercase__     ,		lowercase__     ,		lowercase__:  Optional[Any]         =  config_and_inputs
        lowercase__:  Optional[int]         =  {'pixel_values': pixel_values}
        return config, inputs_dict





@require_tf
class     __a     (       __UpperCamelCase			,			__UpperCamelCase			,			unittest.TestCase ):
 __lowercase					:				List[str]  	=					(
     (
         TFSwinModel,
         TFSwinForImageClassification,
         TFSwinForMaskedImageModeling,
     )
     if is_tf_available()
     else ()
 )
 __lowercase					:				List[str]  	=					(
     {'feature-extraction': TFSwinModel, 'image-classification': TFSwinForImageClassification}
     if is_tf_available()
     else {}
 )

 __lowercase					:				int  	=					False
 __lowercase					:				Tuple  	=					False
 __lowercase					:				Union[str, Any]  	=					False
 __lowercase					:				Any  	=					False
 def 						SCREAMING_SNAKE_CASE__  (  self					)   ->							Tuple:




        '''simple docstring'''
        lowercase__:  Dict         =  TFSwinModelTester(self					)
        lowercase__:  List[str]         =  ConfigTester(self							,					config_class=lowerCAmelCase__							,					embed_dim=37					)
 def 						SCREAMING_SNAKE_CASE__  (  self					)   ->							Any:




        '''simple docstring'''
        self.create_and_test_config_common_properties()
        self.config_tester.create_and_test_config_to_json_string()
        self.config_tester.create_and_test_config_to_json_file()
        self.config_tester.create_and_test_config_from_and_save_pretrained()
        self.config_tester.create_and_test_config_with_num_labels()
        self.config_tester.check_config_can_be_init_without_params()
        self.config_tester.check_config_arguments_init()
 def 						SCREAMING_SNAKE_CASE__  (  self					)   ->							str:




        '''simple docstring'''
        return
 def 						SCREAMING_SNAKE_CASE__  (  self					)   ->							List[str]:




        '''simple docstring'''
        lowercase__:  Dict         =  self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*lowerCAmelCase__					)
 def 						SCREAMING_SNAKE_CASE__  (  self					)   ->							Optional[Any]:




        '''simple docstring'''
        lowercase__:  Optional[Any]         =  self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_masked_image_modeling(*lowerCAmelCase__					)
 def 						SCREAMING_SNAKE_CASE__  (  self					)   ->							List[Any]:




        '''simple docstring'''
        lowercase__:  Tuple         =  self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_image_classification(*lowerCAmelCase__					)
 @unittest.skip(reason='Swin does not use inputs_embeds'					)
 def 						SCREAMING_SNAKE_CASE__  (  self					)   ->							Optional[int]:




        '''simple docstring'''
        pass
 def 						SCREAMING_SNAKE_CASE__  (  self					)   ->							List[Any]:




        '''simple docstring'''
        lowercase__     ,		lowercase__:  int         =  self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
               lowercase__:  Union[str, Any]         =  model_class(lowerCAmelCase__					)
               self.assertIsInstance(model.get_input_embeddings()							,					tf.keras.layers.Layer					)
               lowercase__:  Tuple         =  model.get_output_embeddings()
               self.assertTrue(x is None or isinstance(lowerCAmelCase__							,					tf.keras.layers.Dense					)					)
 def 						SCREAMING_SNAKE_CASE__  (  self					)   ->							List[str]:




        '''simple docstring'''
        lowercase__     ,		lowercase__:  Optional[int]         =  self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
               lowercase__:  Tuple         =  model_class(lowerCAmelCase__					)
               lowercase__:  List[Any]         =  inspect.signature(model.call					)
               # signature.parameters is an OrderedDict => so arg_names order is deterministic
               lowercase__:  List[str]         =  [*signature.parameters.keys()]

               lowercase__:  Union[str, Any]         =  ['pixel_values']
               self.assertListEqual(arg_names[:1]							,					lowerCAmelCase__					)
 def 						SCREAMING_SNAKE_CASE__  (  self					)   ->							Union[str, Any]:




        '''simple docstring'''
        lowercase__     ,		lowercase__:  int         =  self.model_tester.prepare_config_and_inputs_for_common()
        lowercase__:  int         =  True

        for model_class in self.all_model_classes:
               lowercase__:  Optional[int]         =  True
               lowercase__:  Optional[Any]         =  False
               lowercase__:  List[Any]         =  True
               lowercase__:  Any         =  model_class(lowerCAmelCase__					)
               lowercase__:  List[Any]         =  model(**self._prepare_for_class(lowerCAmelCase__							,					lowerCAmelCase__					)					)
               lowercase__:  Optional[Any]         =  outputs.attentions
               lowercase__:  Union[str, Any]         =  len(self.model_tester.depths					)
               self.assertEqual(len(lowerCAmelCase__					)							,					lowerCAmelCase__					)

               # check that output_attentions also work using config
               del inputs_dict["output_attentions"]
               lowercase__:  Optional[int]         =  True
               lowercase__:  Optional[int]         =  config.window_size**2
               lowercase__:  List[Any]         =  model_class(lowerCAmelCase__					)
               lowercase__:  Dict         =  model(**self._prepare_for_class(lowerCAmelCase__							,					lowerCAmelCase__					)					)
               lowercase__:  Tuple         =  outputs.attentions
               self.assertEqual(len(lowerCAmelCase__					)							,					lowerCAmelCase__					)

               self.assertListEqual(
                   list(attentions[0].shape[-3:]					)							,					[self.model_tester.num_heads[0], window_size_squared, window_size_squared]							,					)
               lowercase__:  Any         =  len(lowerCAmelCase__					)

               # Check attention is always last and order is fine
               lowercase__:  Optional[int]         =  True
               lowercase__:  Any         =  True
               lowercase__:  Dict         =  model_class(lowerCAmelCase__					)
               lowercase__:  str         =  model(**self._prepare_for_class(lowerCAmelCase__							,					lowerCAmelCase__					)					)

               if hasattr(self.model_tester							,					'num_hidden_states_types'					):
                      lowercase__:  Tuple         =  self.model_tester.num_hidden_states_types
               else:
                      # also another +1 for reshaped_hidden_states
                      lowercase__:  Union[str, Any]         =  2
               self.assertEqual(out_len + added_hidden_states							,					len(lowerCAmelCase__					)					)

               lowercase__:  Any         =  outputs.attentions

               self.assertEqual(len(lowerCAmelCase__					)							,					lowerCAmelCase__					)

               self.assertListEqual(
                   list(self_attentions[0].shape[-3:]					)							,					[self.model_tester.num_heads[0], window_size_squared, window_size_squared]							,					)
 def 						SCREAMING_SNAKE_CASE__  (  self							,					lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__					)   ->							Tuple:




        '''simple docstring'''
        lowercase__:  List[Any]         =  model_class(lowerCAmelCase__					)
        lowercase__:  Optional[int]         =  model(**self._prepare_for_class(lowerCAmelCase__							,					lowerCAmelCase__					)					)
        lowercase__:  List[str]         =  outputs.hidden_states

        lowercase__:  List[str]         =  getattr(
            self.model_tester							,					'expected_num_hidden_layers'							,					len(self.model_tester.depths					) + 1					)
        self.assertEqual(len(lowerCAmelCase__					)							,					lowerCAmelCase__					)

        # Swin has a different seq_length
        lowercase__:  Tuple         =  to_atuple(config.patch_size					)

        lowercase__:  Dict         =  (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

        self.assertListEqual(
            list(hidden_states[0].shape[-2:]					)							,					[num_patches, self.model_tester.embed_dim]							,					)

        lowercase__:  int         =  outputs.reshaped_hidden_states
        self.assertEqual(len(lowerCAmelCase__					)							,					lowerCAmelCase__					)

        lowercase__     ,		lowercase__     ,		lowercase__     ,		lowercase__:  List[str]         =  reshaped_hidden_states[0].shape

        lowercase__:  List[str]         =  tf.reshape(reshaped_hidden_states[0]							,					(batch_size, num_channels, height * width)					)
        lowercase__:  str         =  tf.transpose(lowerCAmelCase__							,					(0, 2, 1)					)

        self.assertListEqual(
            list(reshaped_hidden_states.shape[-2:]					)							,					[num_patches, self.model_tester.embed_dim]							,					)
 def 						SCREAMING_SNAKE_CASE__  (  self					)   ->							List[str]:




        '''simple docstring'''
        lowercase__     ,		lowercase__:  List[Any]         =  self.model_tester.prepare_config_and_inputs_for_common()

        lowercase__:  List[Any]         =  to_atuple(self.model_tester.image_size					)

        for model_class in self.all_model_classes:
               lowercase__:  Optional[int]         =  True
               self.check_hidden_states_output(lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__					)

               # check that output_hidden_states also work using config
               del inputs_dict["output_hidden_states"]
               lowercase__:  Union[str, Any]         =  True

               self.check_hidden_states_output(lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__					)
 def 						SCREAMING_SNAKE_CASE__  (  self					)   ->							Any:




        '''simple docstring'''
        lowercase__     ,		lowercase__:  int         =  self.model_tester.prepare_config_and_inputs_for_common()
        lowercase__:  Tuple         =  3

        lowercase__:  List[Any]         =  to_atuple(self.model_tester.image_size					)
        lowercase__:  Union[str, Any]         =  to_atuple(config.patch_size					)

        lowercase__:  List[Any]         =  image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
        lowercase__:  Tuple         =  image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

        for model_class in self.all_model_classes:
               lowercase__:  List[Any]         =  True
               self.check_hidden_states_output(lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__							,					(padded_height, padded_width)					)

               # check that output_hidden_states also work using config
               del inputs_dict["output_hidden_states"]
               lowercase__:  List[str]         =  True
               self.check_hidden_states_output(lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__							,					(padded_height, padded_width)					)
 @slow
 def 						SCREAMING_SNAKE_CASE__  (  self					)   ->							int:




        '''simple docstring'''
        for model_name in TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               lowercase__:  Optional[Any]         =  TFSwinModel.from_pretrained(lowerCAmelCase__					)
               self.assertIsNotNone(lowerCAmelCase__					)





@require_vision
@require_tf
class     __a     (       unittest.TestCase ):
 @cached_property
 def 						SCREAMING_SNAKE_CASE__  (  self					)   ->							Any:




        '''simple docstring'''
        return (
            AutoImageProcessor.from_pretrained('microsoft/swin-tiny-patch4-window7-224'					)
            if is_vision_available()
            else None
        )
 @slow
 def 						SCREAMING_SNAKE_CASE__  (  self					)   ->							int:




        '''simple docstring'''
        lowercase__:  List[Any]         =  TFSwinForImageClassification.from_pretrained('microsoft/swin-tiny-patch4-window7-224'					)
        lowercase__:  Optional[int]         =  self.default_image_processor

        lowercase__:  List[str]         =  Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'					)
        lowercase__:  Any         =  image_processor(images=lowerCAmelCase__							,					return_tensors='tf'					)

        # forward pass
        lowercase__:  Dict         =  model(lowerCAmelCase__					)

        # verify the logits
        lowercase__:  List[str]         =  tf.TensorShape((1, 1_000)					)
        self.assertEqual(outputs.logits.shape							,					lowerCAmelCase__					)
        lowercase__:  List[str]         =  tf.constant([-0.0_9_4_8, -0.6_4_5_4, -0.0_9_2_1]					)
        self.assertTrue(np.allclose(outputs.logits[0, :3]							,					lowerCAmelCase__							,					atol=1E-4					)					)






import datasets

from .evaluate import (
    apply_no_ans_threshold,
    find_all_best_thresh,
    get_raw_scores,
    make_eval_dict,
    make_qid_to_has_ans,
    merge_eval,
)


__lowerCAmelCase									=				'''\
@inproceedings{Rajpurkar2016SQuAD10,
  title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text},
  author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang},
  booktitle={EMNLP},
  year={2016}
}
'''

__lowerCAmelCase									=				'''
This metric wrap the official scoring script for version 2 of the Stanford Question
Answering Dataset (SQuAD).

Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by
crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span,
from the corresponding reading passage, or the question might be unanswerable.

SQuAD2.0 combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions
written adversarially by crowdworkers to look similar to answerable ones.
To do well on SQuAD2.0, systems must not only answer questions when possible, but also
determine when no answer is supported by the paragraph and abstain from answering.
'''

__lowerCAmelCase									=				'''
Computes SQuAD v2 scores (F1 and EM).
Args:
    predictions: List of triple for question-answers to score with the following elements:
        - the question-answer \'id\' field as given in the references (see below)
        - the text of the answer
        - the probability that the question has no answer
    references: List of question-answers dictionaries with the following key-values:
            - \'id\': id of the question-answer pair (see above),
            - \'answers\': a list of Dict {\'text\': text of the answer as a string}
    no_answer_threshold: float
        Probability threshold to decide that a question has no answer.
Returns:
    \'exact\': Exact match (the normalized answer exactly match the gold answer)
    \'f1\': The F-score of predicted tokens versus the gold answer
    \'total\': Number of score considered
    \'HasAns_exact\': Exact match (the normalized answer exactly match the gold answer)
    \'HasAns_f1\': The F-score of predicted tokens versus the gold answer
    \'HasAns_total\': Number of score considered
    \'NoAns_exact\': Exact match (the normalized answer exactly match the gold answer)
    \'NoAns_f1\': The F-score of predicted tokens versus the gold answer
    \'NoAns_total\': Number of score considered
    \'best_exact\': Best exact match (with varying threshold)
    \'best_exact_thresh\': No-answer probability threshold associated to the best exact match
    \'best_f1\': Best F1 (with varying threshold)
    \'best_f1_thresh\': No-answer probability threshold associated to the best F1
Examples:

    >>> predictions = [{\'prediction_text\': \'1976\', \'id\': \'56e10a3be3433e1400422b22\', \'no_answer_probability\': 0.}]
    >>> references = [{\'answers\': {\'answer_start\': [97], \'text\': [\'1976\']}, \'id\': \'56e10a3be3433e1400422b22\'}]
    >>> squad_v2_metric = datasets.load_metric("squad_v2")
    >>> results = squad_v2_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact\': 100.0, \'f1\': 100.0, \'total\': 1, \'HasAns_exact\': 100.0, \'HasAns_f1\': 100.0, \'HasAns_total\': 1, \'best_exact\': 100.0, \'best_exact_thresh\': 0.0, \'best_f1\': 100.0, \'best_f1_thresh\': 0.0}
'''





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION			,			_KWARGS_DESCRIPTION )
class     __a     (       datasets.Metric ):
 def 						SCREAMING_SNAKE_CASE__  (  self					)   ->							Dict:




        '''simple docstring'''
        return datasets.MetricInfo(
            description=_DESCRIPTION							,					citation=_CITATION							,					inputs_description=_KWARGS_DESCRIPTION							,					features=datasets.Features(
                {
                    'predictions': {
                        'id': datasets.Value('string'					),
                        'prediction_text': datasets.Value('string'					),
                        'no_answer_probability': datasets.Value('float32'					),
                    },
                    'references': {
                        'id': datasets.Value('string'					),
                        'answers': datasets.features.Sequence(
                            {'text': datasets.Value('string'					), 'answer_start': datasets.Value('int32'					)}					),
                    },
                }					)							,					codebase_urls=['https://rajpurkar.github.io/SQuAD-explorer/']							,					reference_urls=['https://rajpurkar.github.io/SQuAD-explorer/']							,					)
 def 						SCREAMING_SNAKE_CASE__  (  self							,					lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__=1.0					)   ->							str:




        '''simple docstring'''
        lowercase__:  int         =  {p['id']: p['no_answer_probability'] for p in predictions}
        lowercase__:  int         =  [{'paragraphs': [{'qas': references}]}]
        lowercase__:  Optional[int]         =  {p['id']: p['prediction_text'] for p in predictions}

        lowercase__:  Any         =  make_qid_to_has_ans(lowerCAmelCase__					)  # maps qid to True/False
        lowercase__:  Optional[Any]         =  [k for k, v in qid_to_has_ans.items() if v]
        lowercase__:  Dict         =  [k for k, v in qid_to_has_ans.items() if not v]

        lowercase__     ,		lowercase__:  Dict         =  get_raw_scores(lowerCAmelCase__							,					lowerCAmelCase__					)
        lowercase__:  Optional[int]         =  apply_no_ans_threshold(lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__					)
        lowercase__:  List[Any]         =  apply_no_ans_threshold(lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__					)
        lowercase__:  Dict         =  make_eval_dict(lowerCAmelCase__							,					lowerCAmelCase__					)

        if has_ans_qids:
               lowercase__:  List[Any]         =  make_eval_dict(lowerCAmelCase__							,					lowerCAmelCase__							,					qid_list=lowerCAmelCase__					)
               merge_eval(lowerCAmelCase__							,					lowerCAmelCase__							,					'HasAns'					)
        if no_ans_qids:
               lowercase__:  int         =  make_eval_dict(lowerCAmelCase__							,					lowerCAmelCase__							,					qid_list=lowerCAmelCase__					)
               merge_eval(lowerCAmelCase__							,					lowerCAmelCase__							,					'NoAns'					)
        find_all_best_thresh(lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__					)
        return dict(lowerCAmelCase__					)


