






'''simple docstring'''







import unittest

from transformers import AutoTokenizer, NystromformerConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    NystromformerForMaskedLM,
			    NystromformerForMultipleChoice,
			    NystromformerForQuestionAnswering,
			    NystromformerForSequenceClassification,
			    NystromformerForTokenClassification,
			    NystromformerModel,
			)
			from transformers.models.nystromformer.modeling_nystromformer import NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST

class _lowercase	:




							def __init__(							self:				Union[str, Any]			,							UpperCamelCase__:				Tuple			,							UpperCamelCase__:				List[str]=13			,							UpperCamelCase__:				Optional[Any]=7			,							UpperCamelCase__:				Optional[int]=True			,							UpperCamelCase__:				List[str]=True			,							UpperCamelCase__:				Optional[Any]=True			,							UpperCamelCase__:				str=True			,							UpperCamelCase__:				Dict=99			,							UpperCamelCase__:				Optional[Any]=32			,							UpperCamelCase__:				Dict=5			,							UpperCamelCase__:				List[Any]=4			,							UpperCamelCase__:				Optional[int]=37			,							UpperCamelCase__:				str="gelu"			,							UpperCamelCase__:				List[Any]=0.1			,							UpperCamelCase__:				Optional[Any]=0.1			,							UpperCamelCase__:				Optional[Any]=512			,							UpperCamelCase__:				Optional[Any]=16			,							UpperCamelCase__:				str=2			,							UpperCamelCase__:				Any=0.02			,							UpperCamelCase__:				Tuple=3			,							UpperCamelCase__:				Optional[Any]=4			,							UpperCamelCase__:				Union[str, Any]=None			,							):
														lowerCamelCase__					:	Optional[Any]         =       parent
														lowerCamelCase__					:	str         =       batch_size
														lowerCamelCase__					:	Optional[int]         =       seq_length
														lowerCamelCase__					:	Any         =       is_training
														lowerCamelCase__					:	str         =       use_input_mask
														lowerCamelCase__					:	Tuple         =       use_token_type_ids
														lowerCamelCase__					:	Any         =       use_labels
														lowerCamelCase__					:	int         =       vocab_size
														lowerCamelCase__					:	Dict         =       hidden_size
														lowerCamelCase__					:	str         =       num_hidden_layers
														lowerCamelCase__					:	List[Any]         =       num_attention_heads
														lowerCamelCase__					:	List[Any]         =       intermediate_size
														lowerCamelCase__					:	List[str]         =       hidden_act
														lowerCamelCase__					:	List[Any]         =       hidden_dropout_prob
														lowerCamelCase__					:	Optional[Any]         =       attention_probs_dropout_prob
														lowerCamelCase__					:	List[Any]         =       max_position_embeddings
														lowerCamelCase__					:	str         =       type_vocab_size
														lowerCamelCase__					:	List[Any]         =       type_sequence_label_size
														lowerCamelCase__					:	List[Any]         =       initializer_range
														lowerCamelCase__					:	List[str]         =       num_labels
														lowerCamelCase__					:	List[str]         =       num_choices
														lowerCamelCase__					:	Tuple         =       scope




							def 		lowerCamelCase_				(							self:				str     ):
														lowerCamelCase__					:	Union[str, Any]         =       ids_tensor([self.batch_size, self.seq_length]			,							self.vocab_size     )

														lowerCamelCase__					:	List[Any]         =       None
														if self.use_input_mask:
																					lowerCamelCase__					:	Dict         =       random_attention_mask([self.batch_size, self.seq_length]     )

														lowerCamelCase__					:	str         =       None
														if self.use_token_type_ids:
																					lowerCamelCase__					:	List[str]         =       ids_tensor([self.batch_size, self.seq_length]			,							self.type_vocab_size     )

														lowerCamelCase__					:	str         =       None
														lowerCamelCase__					:	Optional[Any]         =       None
														lowerCamelCase__					:	Any         =       None
														if self.use_labels:
																					lowerCamelCase__					:	List[Any]         =       ids_tensor([self.batch_size]			,							self.type_sequence_label_size     )
																					lowerCamelCase__					:	Union[str, Any]         =       ids_tensor([self.batch_size, self.seq_length]			,							self.num_labels     )
																					lowerCamelCase__					:	Any         =       ids_tensor([self.batch_size]			,							self.num_choices     )

														lowerCamelCase__					:	str         =       self.get_config()

														return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




							def 		lowerCamelCase_				(							self:				str     ):
														return NystromformerConfig(
														    vocab_size=self.vocab_size			,							hidden_size=self.hidden_size			,							num_hidden_layers=self.num_hidden_layers			,							num_attention_heads=self.num_attention_heads			,							intermediate_size=self.intermediate_size			,							hidden_act=self.hidden_act			,							hidden_dropout_prob=self.hidden_dropout_prob			,							attention_probs_dropout_prob=self.attention_probs_dropout_prob			,							max_position_embeddings=self.max_position_embeddings			,							type_vocab_size=self.type_vocab_size			,							is_decoder=UpperCamelCase__			,							initializer_range=self.initializer_range			,							)




							def 		lowerCamelCase_				(							self:				Optional[int]			,							UpperCamelCase__:				List[str]			,							UpperCamelCase__:				Tuple			,							UpperCamelCase__:				Tuple			,							UpperCamelCase__:				Any			,							UpperCamelCase__:				Tuple			,							UpperCamelCase__:				List[str]			,							UpperCamelCase__:				int     ):
														lowerCamelCase__					:	Union[str, Any]         =       NystromformerModel(config=UpperCamelCase__     )
														model.to(UpperCamelCase__     )
														model.eval()
														lowerCamelCase__					:	Optional[Any]         =       model(UpperCamelCase__			,							attention_mask=UpperCamelCase__			,							token_type_ids=UpperCamelCase__     )
														lowerCamelCase__					:	Optional[int]         =       model(UpperCamelCase__			,							token_type_ids=UpperCamelCase__     )
														lowerCamelCase__					:	List[str]         =       model(UpperCamelCase__     )
														self.parent.assertEqual(result.last_hidden_state.shape			,							(self.batch_size, self.seq_length, self.hidden_size)     )




							def 		lowerCamelCase_				(							self:				Tuple			,							UpperCamelCase__:				Any			,							UpperCamelCase__:				Optional[Any]			,							UpperCamelCase__:				Union[str, Any]			,							UpperCamelCase__:				Optional[Any]			,							UpperCamelCase__:				Any			,							UpperCamelCase__:				Optional[Any]			,							UpperCamelCase__:				Tuple     ):
														lowerCamelCase__					:	Any         =       NystromformerForMaskedLM(config=UpperCamelCase__     )
														model.to(UpperCamelCase__     )
														model.eval()
														lowerCamelCase__					:	Optional[int]         =       model(UpperCamelCase__			,							attention_mask=UpperCamelCase__			,							token_type_ids=UpperCamelCase__			,							labels=UpperCamelCase__     )
														self.parent.assertEqual(result.logits.shape			,							(self.batch_size, self.seq_length, self.vocab_size)     )




							def 		lowerCamelCase_				(							self:				List[str]			,							UpperCamelCase__:				Any			,							UpperCamelCase__:				str			,							UpperCamelCase__:				Optional[int]			,							UpperCamelCase__:				str			,							UpperCamelCase__:				Union[str, Any]			,							UpperCamelCase__:				Optional[int]			,							UpperCamelCase__:				Optional[Any]     ):
														lowerCamelCase__					:	Optional[Any]         =       NystromformerForQuestionAnswering(config=UpperCamelCase__     )
														model.to(UpperCamelCase__     )
														model.eval()
														lowerCamelCase__					:	int         =       model(
														    UpperCamelCase__			,							attention_mask=UpperCamelCase__			,							token_type_ids=UpperCamelCase__			,							start_positions=UpperCamelCase__			,							end_positions=UpperCamelCase__			,							)
														self.parent.assertEqual(result.start_logits.shape			,							(self.batch_size, self.seq_length)     )
														self.parent.assertEqual(result.end_logits.shape			,							(self.batch_size, self.seq_length)     )




							def 		lowerCamelCase_				(							self:				str			,							UpperCamelCase__:				str			,							UpperCamelCase__:				Union[str, Any]			,							UpperCamelCase__:				int			,							UpperCamelCase__:				List[Any]			,							UpperCamelCase__:				Optional[int]			,							UpperCamelCase__:				Union[str, Any]			,							UpperCamelCase__:				Optional[Any]     ):
														lowerCamelCase__					:	List[Any]         =       self.num_labels
														lowerCamelCase__					:	Tuple         =       NystromformerForSequenceClassification(UpperCamelCase__     )
														model.to(UpperCamelCase__     )
														model.eval()
														lowerCamelCase__					:	int         =       model(UpperCamelCase__			,							attention_mask=UpperCamelCase__			,							token_type_ids=UpperCamelCase__			,							labels=UpperCamelCase__     )
														self.parent.assertEqual(result.logits.shape			,							(self.batch_size, self.num_labels)     )




							def 		lowerCamelCase_				(							self:				List[str]			,							UpperCamelCase__:				Union[str, Any]			,							UpperCamelCase__:				Union[str, Any]			,							UpperCamelCase__:				List[str]			,							UpperCamelCase__:				Union[str, Any]			,							UpperCamelCase__:				Union[str, Any]			,							UpperCamelCase__:				Tuple			,							UpperCamelCase__:				List[Any]     ):
														lowerCamelCase__					:	Tuple         =       self.num_labels
														lowerCamelCase__					:	Dict         =       NystromformerForTokenClassification(config=UpperCamelCase__     )
														model.to(UpperCamelCase__     )
														model.eval()
														lowerCamelCase__					:	int         =       model(UpperCamelCase__			,							attention_mask=UpperCamelCase__			,							token_type_ids=UpperCamelCase__			,							labels=UpperCamelCase__     )
														self.parent.assertEqual(result.logits.shape			,							(self.batch_size, self.seq_length, self.num_labels)     )




							def 		lowerCamelCase_				(							self:				List[Any]			,							UpperCamelCase__:				List[str]			,							UpperCamelCase__:				Dict			,							UpperCamelCase__:				Tuple			,							UpperCamelCase__:				int			,							UpperCamelCase__:				Any			,							UpperCamelCase__:				Union[str, Any]			,							UpperCamelCase__:				int     ):
														lowerCamelCase__					:	Optional[Any]         =       self.num_choices
														lowerCamelCase__					:	Any         =       NystromformerForMultipleChoice(config=UpperCamelCase__     )
														model.to(UpperCamelCase__     )
														model.eval()
														lowerCamelCase__					:	Optional[Any]         =       input_ids.unsqueeze(1     ).expand(-1			,							self.num_choices			,							-1     ).contiguous()
														lowerCamelCase__					:	Any         =       token_type_ids.unsqueeze(1     ).expand(-1			,							self.num_choices			,							-1     ).contiguous()
														lowerCamelCase__					:	str         =       input_mask.unsqueeze(1     ).expand(-1			,							self.num_choices			,							-1     ).contiguous()
														lowerCamelCase__					:	Optional[int]         =       model(
														    UpperCamelCase__			,							attention_mask=UpperCamelCase__			,							token_type_ids=UpperCamelCase__			,							labels=UpperCamelCase__			,							)
														self.parent.assertEqual(result.logits.shape			,							(self.batch_size, self.num_choices)     )




							def 		lowerCamelCase_				(							self:				str     ):
														lowerCamelCase__					:	Optional[Any]         =       self.prepare_config_and_inputs()
														(
														    (
														    lowerCamelCase__
														)  ,		(
														    lowerCamelCase__
														)  ,		(
														    lowerCamelCase__
														)  ,		(
														    lowerCamelCase__
														)  ,		(
														    lowerCamelCase__
														)  ,		(
														    lowerCamelCase__
														)  ,		(
														    lowerCamelCase__
														)  ,		
														)					:	Optional[Any]         =       config_and_inputs
														lowerCamelCase__					:	int         =       {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
														return config, inputs_dict



@require_torch
class _lowercase	(				lowercase_				,      lowercase_				,      unittest.TestCase		):
							a				      =       (
							    (
							        NystromformerModel,
							        NystromformerForMaskedLM,
							        NystromformerForMultipleChoice,
							        NystromformerForQuestionAnswering,
							        NystromformerForSequenceClassification,
							        NystromformerForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							a				      =       (
							    {
							        """feature-extraction""": NystromformerModel,
							        """fill-mask""": NystromformerForMaskedLM,
							        """question-answering""": NystromformerForQuestionAnswering,
							        """text-classification""": NystromformerForSequenceClassification,
							        """token-classification""": NystromformerForTokenClassification,
							        """zero-shot""": NystromformerForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							a				      =       False
							a				      =       False




							def 		lowerCamelCase_				(							self:				List[Any]     ):
														lowerCamelCase__					:	Any         =       NystromformerModelTester(self     )
														lowerCamelCase__					:	Dict         =       ConfigTester(self			,							config_class=UpperCamelCase__			,							hidden_size=37     )




							def 		lowerCamelCase_				(							self:				List[str]     ):
														self.config_tester.run_common_tests()




							def 		lowerCamelCase_				(							self:				Tuple     ):
														lowerCamelCase__					:	Any         =       self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_model(*UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				int     ):
														lowerCamelCase__					:	Dict         =       self.model_tester.prepare_config_and_inputs()
														for type in ["absolute", "relative_key", "relative_key_query"]:
																					lowerCamelCase__					:	Any         =       type
																					self.model_tester.create_and_check_model(*UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				int     ):
														lowerCamelCase__					:	List[str]         =       self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_masked_lm(*UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				Optional[int]     ):
														lowerCamelCase__					:	List[Any]         =       self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_multiple_choice(*UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				Optional[Any]     ):
														lowerCamelCase__					:	Any         =       self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_question_answering(*UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				Tuple     ):
														lowerCamelCase__					:	Optional[Any]         =       self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_sequence_classification(*UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				Dict     ):
														lowerCamelCase__					:	Optional[int]         =       self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_token_classification(*UpperCamelCase__     )




							@slow
							def 		lowerCamelCase_				(							self:				List[str]     ):







														for model_name in NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																					lowerCamelCase__					:	int         =       NystromformerModel.from_pretrained(UpperCamelCase__     )
																					self.assertIsNotNone(UpperCamelCase__     )



@require_torch
class _lowercase	(				unittest.TestCase		):




							@slow
							def 		lowerCamelCase_				(							self:				Union[str, Any]     ):
														lowerCamelCase__					:	List[Any]         =       NystromformerModel.from_pretrained("""uw-madison/nystromformer-512"""     )
														lowerCamelCase__					:	Tuple         =       torch.tensor([[0, 1, 2, 3, 4, 5]]     )

														with torch.no_grad():
																					lowerCamelCase__					:	Optional[Any]         =       model(UpperCamelCase__     )[0]

														lowerCamelCase__					:	int         =       torch.Size((1, 6, 768)     )
														self.assertEqual(output.shape			,							UpperCamelCase__     )

														lowerCamelCase__					:	Any         =       torch.tensor(
														    [[[-0.4_532, -0.0_936, 0.5_137], [-0.2_676, 0.0_628, 0.6_186], [-0.3_629, -0.1_726, 0.4_716]]]     )

														self.assertTrue(torch.allclose(output[:, :3, :3]			,							UpperCamelCase__			,							atol=1e-4     )     )




							@slow
							def 		lowerCamelCase_				(							self:				Any     ):
														lowerCamelCase__					:	Optional[Any]         =       """the [MASK] of Belgium is Brussels"""

														lowerCamelCase__					:	str         =       AutoTokenizer.from_pretrained("""uw-madison/nystromformer-512"""     )
														lowerCamelCase__					:	List[str]         =       NystromformerForMaskedLM.from_pretrained("""uw-madison/nystromformer-512"""     )

														lowerCamelCase__					:	str         =       tokenizer(UpperCamelCase__			,							return_tensors="""pt"""     )

														with torch.no_grad():
																					lowerCamelCase__					:	List[str]         =       model(encoding.input_ids     ).logits

														lowerCamelCase__					:	Optional[Any]         =       token_logits[:, 2, :].argmax(-1     )[0]

														self.assertEqual(tokenizer.decode(UpperCamelCase__     )			,							"""capital"""     )





'''simple docstring'''

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase							:   Tuple  						=	logging.get_logger(__name__)

lowerCamelCase							:   Dict  						=	{
    'microsoft/cvt-13': 'https://huggingface.co/microsoft/cvt-13/resolve/main/config.json',
    # See all Cvt models at https://huggingface.co/models?filter=cvt
}
class   __lowerCAmelCase						(lowercase_     ):





      '''simple docstring'''






      lowerCAmelCase__					:     Tuple  					=							"""cvt"""



      def __init__(self    :    int   ,						UpperCamelCase    :    List[Any]=3   ,						UpperCamelCase    :    int=[7, 3, 3]   ,						UpperCamelCase    :    str=[4, 2, 2]   ,						UpperCamelCase    :    Dict=[2, 1, 1]   ,						UpperCamelCase    :    Dict=[64, 192, 384]   ,						UpperCamelCase    :    Dict=[1, 3, 6]   ,						UpperCamelCase    :    Dict=[1, 2, 10]   ,						UpperCamelCase    :    Any=[4.0, 4.0, 4.0]   ,						UpperCamelCase    :    Optional[int]=[0.0, 0.0, 0.0]   ,						UpperCamelCase    :    Optional[int]=[0.0, 0.0, 0.0]   ,						UpperCamelCase    :    int=[0.0, 0.0, 0.1]   ,						UpperCamelCase    :    Any=[True, True, True]   ,						UpperCamelCase    :    int=[False, False, True]   ,						UpperCamelCase    :    Union[str, Any]=["dw_bn", "dw_bn", "dw_bn"]   ,						UpperCamelCase    :    Optional[int]=[3, 3, 3]   ,						UpperCamelCase    :    Tuple=[1, 1, 1]   ,						UpperCamelCase    :    Any=[2, 2, 2]   ,						UpperCamelCase    :    Dict=[1, 1, 1]   ,						UpperCamelCase    :    List[str]=[1, 1, 1]   ,						UpperCamelCase    :    str=0.02   ,						UpperCamelCase    :    int=1E-12   ,						**UpperCamelCase    :    Union[str, Any]   ,						):

             '''simple docstring'''







             super().__init__(**UpperCamelCase )
             lowercase__      			= num_channels
             lowercase__      			= patch_sizes
             lowercase__      			= patch_stride
             lowercase__      			= patch_padding
             lowercase__      			= embed_dim
             lowercase__      			= num_heads
             lowercase__      			= depth
             lowercase__      			= mlp_ratio
             lowercase__      			= attention_drop_rate
             lowercase__      			= drop_rate
             lowercase__      			= drop_path_rate
             lowercase__      			= qkv_bias
             lowercase__      			= cls_token
             lowercase__      			= qkv_projection_method
             lowercase__      			= kernel_qkv
             lowercase__      			= padding_kv
             lowercase__      			= stride_kv
             lowercase__      			= padding_q
             lowercase__      			= stride_q
             lowercase__      			= initializer_range
             lowercase__      			= layer_norm_eps




