"""simple docstring"""
import torch

from diffusers import KDPMaAncestralDiscreteScheduler
from diffusers.utils import torch_device

from .test_schedulers import SchedulerCommonTest



class 	lowerCAmelCase_     (      lowerCamelCase__       ):



				"""simple docstring"""
				_lowerCAmelCase  :     Tuple      	=    (KDPMaAncestralDiscreteScheduler,)
				_lowerCAmelCase  :     Optional[int]      	=    10

				def 		snake_case			(					self		,     **lowerCAmelCase     ):






										"""simple docstring"""

										snake_case             =      {
										    '''num_train_timesteps''': 11_00,
										    '''beta_start''': 0.00_01,
										    '''beta_end''': 0.02,
										    '''beta_schedule''': '''linear''',
										}

										config.update(**__A     )
										return config

				def 		snake_case			(					self     ):






										"""simple docstring"""

										for timesteps in [10, 50, 1_00, 10_00]:
																self.check_over_configs(num_train_timesteps=__A     )

				def 		snake_case			(					self     ):






										"""simple docstring"""

										for beta_start, beta_end in zip([0.0_00_01, 0.00_01, 0.0_01]		,     [0.00_02, 0.0_02, 0.02]     ):
																self.check_over_configs(beta_start=__A		,     beta_end=__A     )

				def 		snake_case			(					self     ):






										"""simple docstring"""

										for schedule in ["linear", "scaled_linear"]:
																self.check_over_configs(beta_schedule=__A     )

				def 		snake_case			(					self     ):






										"""simple docstring"""

										if torch_device == "mps":
																return
										snake_case             =      self.scheduler_classes[0]
										snake_case             =      self.get_scheduler_config()
										snake_case             =      scheduler_class(**__A     )

										scheduler.set_timesteps(self.num_inference_steps     )

										snake_case             =      torch.manual_seed(0     )

										snake_case             =      self.dummy_model()
										snake_case             =      self.dummy_sample_deter * scheduler.init_noise_sigma
										snake_case             =      sample.to(__A     )

										for i, t in enumerate(scheduler.timesteps     ):
																snake_case             =      scheduler.scale_model_input(__A		,     __A     )

																snake_case             =      model(__A		,     __A     )

																snake_case             =      scheduler.step(__A		,     __A		,     __A		,     generator=__A     )
																snake_case             =      output.prev_sample

										snake_case             =      torch.sum(torch.abs(__A     )     )
										snake_case             =      torch.mean(torch.abs(__A     )     )

										assert abs(result_sum.item() - 1_38_49.38_77     ) < 1E-2
										assert abs(result_mean.item() - 18.03_31     ) < 5E-3

				def 		snake_case			(					self     ):






										"""simple docstring"""

										for prediction_type in ["epsilon", "v_prediction"]:
																self.check_over_configs(prediction_type=__A     )

				def 		snake_case			(					self     ):






										"""simple docstring"""

										if torch_device == "mps":
																return
										snake_case             =      self.scheduler_classes[0]
										snake_case             =      self.get_scheduler_config(prediction_type='v_prediction'     )
										snake_case             =      scheduler_class(**__A     )

										scheduler.set_timesteps(self.num_inference_steps     )

										snake_case             =      self.dummy_model()
										snake_case             =      self.dummy_sample_deter * scheduler.init_noise_sigma
										snake_case             =      sample.to(__A     )

										snake_case             =      torch.manual_seed(0     )

										for i, t in enumerate(scheduler.timesteps     ):
																snake_case             =      scheduler.scale_model_input(__A		,     __A     )

																snake_case             =      model(__A		,     __A     )

																snake_case             =      scheduler.step(__A		,     __A		,     __A		,     generator=__A     )
																snake_case             =      output.prev_sample

										snake_case             =      torch.sum(torch.abs(__A     )     )
										snake_case             =      torch.mean(torch.abs(__A     )     )

										assert abs(result_sum.item() - 3_28.99_70     ) < 1E-2
										assert abs(result_mean.item() - 0.42_84     ) < 1E-3

				def 		snake_case			(					self     ):






										"""simple docstring"""

										if torch_device == "mps":
																return
										snake_case             =      self.scheduler_classes[0]
										snake_case             =      self.get_scheduler_config()
										snake_case             =      scheduler_class(**__A     )

										scheduler.set_timesteps(self.num_inference_steps		,     device=__A     )
										snake_case             =      torch.manual_seed(0     )

										snake_case             =      self.dummy_model()
										snake_case             =      self.dummy_sample_deter.to(__A     ) * scheduler.init_noise_sigma

										for t in scheduler.timesteps:
																snake_case             =      scheduler.scale_model_input(__A		,     __A     )

																snake_case             =      model(__A		,     __A     )

																snake_case             =      scheduler.step(__A		,     __A		,     __A		,     generator=__A     )
																snake_case             =      output.prev_sample

										snake_case             =      torch.sum(torch.abs(__A     )     )
										snake_case             =      torch.mean(torch.abs(__A     )     )

										assert abs(result_sum.item() - 1_38_49.38_18     ) < 1E-1
										assert abs(result_mean.item() - 18.03_31     ) < 1E-3





import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         MODEL_FOR_PRETRAINING_MAPPING,
         FunnelBaseModel,
         FunnelForMaskedLM,
         FunnelForMultipleChoice,
         FunnelForPreTraining,
         FunnelForQuestionAnswering,
         FunnelForSequenceClassification,
         FunnelForTokenClassification,
         FunnelModel,
     )





class    __UpperCAmelCase	:



   def __init__(				self				:	List[Any],						__A				:	str,						__A				:	Union[str, Any]=1_3,						__A				:	Union[str, Any]=7,						__A				:	List[str]=True,						__A				:	Optional[Any]=True,						__A				:	Tuple=True,						__A				:	Any=True,						__A				:	Dict=9_9,						__A				:	Union[str, Any]=[1, 1, 2],						__A				:	Tuple=1,						__A				:	int=3_2,						__A				:	List[str]=4,						__A				:	List[str]=8,						__A				:	int=3_7,						__A				:	Optional[int]="gelu_new",						__A				:	List[Any]=0.1,						__A				:	str=0.1,						__A				:	int=0.0,						__A				:	Union[str, Any]=5_1_2,						__A				:	int=3,						__A				:	str=0.0_2,						__A				:	Tuple=3,						__A				:	Any=4,						__A				:	Any=None,						__A				:	Any=False,						):
     UpperCAmelCase			:				str										=					parent
     UpperCAmelCase			:				Optional[Any]										=					batch_size
     UpperCAmelCase			:				str										=					seq_length
     UpperCAmelCase			:				str										=					is_training
     UpperCAmelCase			:				Tuple										=					use_input_mask
     UpperCAmelCase			:				Union[str, Any]										=					use_token_type_ids
     UpperCAmelCase			:				Dict										=					use_labels
     UpperCAmelCase			:				Union[str, Any]										=					vocab_size
     UpperCAmelCase			:				Tuple										=					block_sizes
     UpperCAmelCase			:				List[Any]										=					num_decoder_layers
     UpperCAmelCase			:				Union[str, Any]										=					d_model
     UpperCAmelCase			:				Tuple										=					n_head
     UpperCAmelCase			:				Dict										=					d_head
     UpperCAmelCase			:				Optional[int]										=					d_inner
     UpperCAmelCase			:				Optional[int]										=					hidden_act
     UpperCAmelCase			:				int										=					hidden_dropout
     UpperCAmelCase			:				Any										=					attention_dropout
     UpperCAmelCase			:				str										=					activation_dropout
     UpperCAmelCase			:				Dict										=					max_position_embeddings
     UpperCAmelCase			:				Optional[Any]										=					type_vocab_size
     UpperCAmelCase			:				Any										=					2
     UpperCAmelCase			:				List[Any]										=					num_labels
     UpperCAmelCase			:				str										=					num_choices
     UpperCAmelCase			:				str										=					scope
     UpperCAmelCase			:				int										=					initializer_std

     # Used in the tests to check the size of the first attention layer
     UpperCAmelCase			:				int										=					n_head
     # Used in the tests to check the size of the first hidden state
     UpperCAmelCase			:				Dict										=					self.d_model
     # Used in the tests to check the number of output hidden states/attentions
     UpperCAmelCase			:				List[str]										=					sum(self.block_sizes		) + (0 if base else self.num_decoder_layers)
     # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
     # the last hidden state of the first block (which is the first hidden state of the decoder).
     if not base:
       UpperCAmelCase			:				List[str]										=					self.num_hidden_layers + 2



   def 			__magic_name__       (				self				:	List[str]		):
     UpperCAmelCase			:				Dict										=					ids_tensor([self.batch_size, self.seq_length],						self.vocab_size		)

     UpperCAmelCase			:				Optional[int]										=					None
     if self.use_input_mask:
       UpperCAmelCase			:				List[str]										=					ids_tensor([self.batch_size, self.seq_length],						vocab_size=2		)

     UpperCAmelCase			:				int										=					None
     if self.use_token_type_ids:
       UpperCAmelCase			:				Dict										=					ids_tensor([self.batch_size, self.seq_length],						self.type_vocab_size		)

     UpperCAmelCase			:				List[Any]										=					None
     UpperCAmelCase			:				Optional[int]										=					None
     UpperCAmelCase			:				str										=					None
     if self.use_labels:
       UpperCAmelCase			:				List[Any]										=					ids_tensor([self.batch_size],						self.type_sequence_label_size		)
       UpperCAmelCase			:				Optional[int]										=					ids_tensor([self.batch_size, self.seq_length],						self.num_labels		)
       UpperCAmelCase			:				List[str]										=					ids_tensor([self.batch_size],						self.num_choices		)
       UpperCAmelCase			:				Tuple										=					ids_tensor([self.batch_size, self.seq_length],						1		)

     UpperCAmelCase			:				Dict										=					self.get_config()

     return (
         config,
         input_ids,
         token_type_ids,
         input_mask,
         sequence_labels,
         token_labels,
         choice_labels,
         fake_token_labels,
     )



   def 			__magic_name__       (				self				:	List[Any]		):
     return FunnelConfig(
         vocab_size=self.vocab_size,						block_sizes=self.block_sizes,						num_decoder_layers=self.num_decoder_layers,						d_model=self.d_model,						n_head=self.n_head,						d_head=self.d_head,						d_inner=self.d_inner,						hidden_act=self.hidden_act,						hidden_dropout=self.hidden_dropout,						attention_dropout=self.attention_dropout,						activation_dropout=self.activation_dropout,						max_position_embeddings=self.max_position_embeddings,						type_vocab_size=self.type_vocab_size,						initializer_std=self.initializer_std,						)



   def 			__magic_name__       (				self				:	Union[str, Any],						__A				:	List[str],						__A				:	Dict,						__A				:	Optional[int],						__A				:	List[Any],						__A				:	List[Any],						__A				:	Optional[int],						__A				:	Dict,						__A				:	Dict,						):
     UpperCAmelCase			:				List[str]										=					FunnelModel(config=__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				List[Any]										=					model(__A,						attention_mask=__A,						token_type_ids=__A		)
     UpperCAmelCase			:				Optional[int]										=					model(__A,						token_type_ids=__A		)
     UpperCAmelCase			:				Optional[int]										=					model(__A		)
     self.parent.assertEqual(result.last_hidden_state.shape,						(self.batch_size, self.seq_length, self.d_model)		)

     UpperCAmelCase			:				Optional[int]										=					False
     UpperCAmelCase			:				List[str]										=					model(__A		)
     self.parent.assertEqual(result.last_hidden_state.shape,						(self.batch_size, self.seq_length, self.d_model)		)

     UpperCAmelCase			:				List[str]										=					False
     UpperCAmelCase			:				Dict										=					model(__A		)
     self.parent.assertEqual(result.last_hidden_state.shape,						(self.batch_size, self.seq_length, self.d_model)		)



   def 			__magic_name__       (				self				:	Dict,						__A				:	str,						__A				:	List[Any],						__A				:	Union[str, Any],						__A				:	Union[str, Any],						__A				:	Tuple,						__A				:	Tuple,						__A				:	List[Any],						__A				:	Dict,						):
     UpperCAmelCase			:				List[Any]										=					FunnelBaseModel(config=__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				int										=					model(__A,						attention_mask=__A,						token_type_ids=__A		)
     UpperCAmelCase			:				List[Any]										=					model(__A,						token_type_ids=__A		)
     UpperCAmelCase			:				Optional[Any]										=					model(__A		)
     self.parent.assertEqual(result.last_hidden_state.shape,						(self.batch_size, 2, self.d_model)		)

     UpperCAmelCase			:				str										=					False
     UpperCAmelCase			:				Optional[Any]										=					model(__A		)
     self.parent.assertEqual(result.last_hidden_state.shape,						(self.batch_size, 3, self.d_model)		)

     UpperCAmelCase			:				Tuple										=					False
     UpperCAmelCase			:				Optional[Any]										=					model(__A		)
     self.parent.assertEqual(result.last_hidden_state.shape,						(self.batch_size, 2, self.d_model)		)



   def 			__magic_name__       (				self				:	Optional[int],						__A				:	Tuple,						__A				:	Dict,						__A				:	str,						__A				:	Dict,						__A				:	Union[str, Any],						__A				:	List[Any],						__A				:	Any,						__A				:	str,						):
     UpperCAmelCase			:				int										=					self.num_labels
     UpperCAmelCase			:				int										=					FunnelForPreTraining(config=__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				List[Any]										=					model(__A,						attention_mask=__A,						token_type_ids=__A,						labels=__A		)
     self.parent.assertEqual(result.logits.shape,						(self.batch_size, self.seq_length)		)



   def 			__magic_name__       (				self				:	List[str],						__A				:	Optional[int],						__A				:	Union[str, Any],						__A				:	int,						__A				:	Union[str, Any],						__A				:	Optional[int],						__A				:	str,						__A				:	int,						__A				:	str,						):
     UpperCAmelCase			:				List[Any]										=					FunnelForMaskedLM(config=__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				Optional[int]										=					model(__A,						attention_mask=__A,						token_type_ids=__A,						labels=__A		)
     self.parent.assertEqual(result.logits.shape,						(self.batch_size, self.seq_length, self.vocab_size)		)



   def 			__magic_name__       (				self				:	Optional[int],						__A				:	Optional[Any],						__A				:	Optional[Any],						__A				:	Dict,						__A				:	Any,						__A				:	Optional[Any],						__A				:	Tuple,						__A				:	Union[str, Any],						__A				:	List[str],						):
     UpperCAmelCase			:				str										=					self.num_labels
     UpperCAmelCase			:				Optional[int]										=					FunnelForSequenceClassification(__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				Dict										=					model(__A,						attention_mask=__A,						token_type_ids=__A,						labels=__A		)
     self.parent.assertEqual(result.logits.shape,						(self.batch_size, self.num_labels)		)



   def 			__magic_name__       (				self				:	Union[str, Any],						__A				:	Any,						__A				:	List[str],						__A				:	str,						__A				:	List[Any],						__A				:	List[str],						__A				:	Optional[Any],						__A				:	List[Any],						__A				:	List[str],						):
     UpperCAmelCase			:				str										=					self.num_choices
     UpperCAmelCase			:				Optional[int]										=					FunnelForMultipleChoice(config=__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				List[Any]										=					input_ids.unsqueeze(1		).expand(-1,						self.num_choices,						-1		).contiguous()
     UpperCAmelCase			:				List[Any]										=					token_type_ids.unsqueeze(1		).expand(-1,						self.num_choices,						-1		).contiguous()
     UpperCAmelCase			:				List[Any]										=					input_mask.unsqueeze(1		).expand(-1,						self.num_choices,						-1		).contiguous()
     UpperCAmelCase			:				List[str]										=					model(
         __A,						attention_mask=__A,						token_type_ids=__A,						labels=__A,						)
     self.parent.assertEqual(result.logits.shape,						(self.batch_size, self.num_choices)		)



   def 			__magic_name__       (				self				:	Dict,						__A				:	Tuple,						__A				:	int,						__A				:	Any,						__A				:	Optional[Any],						__A				:	Optional[Any],						__A				:	Tuple,						__A				:	str,						__A				:	List[str],						):
     UpperCAmelCase			:				Union[str, Any]										=					self.num_labels
     UpperCAmelCase			:				Optional[int]										=					FunnelForTokenClassification(config=__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				Any										=					model(__A,						attention_mask=__A,						token_type_ids=__A,						labels=__A		)
     self.parent.assertEqual(result.logits.shape,						(self.batch_size, self.seq_length, self.num_labels)		)



   def 			__magic_name__       (				self				:	int,						__A				:	Optional[int],						__A				:	List[str],						__A				:	List[Any],						__A				:	int,						__A				:	str,						__A				:	Any,						__A				:	Union[str, Any],						__A				:	str,						):
     UpperCAmelCase			:				str										=					FunnelForQuestionAnswering(config=__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				Optional[Any]										=					model(
         __A,						attention_mask=__A,						token_type_ids=__A,						start_positions=__A,						end_positions=__A,						)
     self.parent.assertEqual(result.start_logits.shape,						(self.batch_size, self.seq_length)		)
     self.parent.assertEqual(result.end_logits.shape,						(self.batch_size, self.seq_length)		)





   def 			__magic_name__       (				self				:	Optional[Any]		):
     UpperCAmelCase			:				Dict										=					self.prepare_config_and_inputs()
     (
         (
         UpperCAmelCase
     )					,		(
         UpperCAmelCase
     )					,		(
         UpperCAmelCase
     )					,		(
         UpperCAmelCase
     )					,		(
         UpperCAmelCase
     )					,		(
         UpperCAmelCase
     )					,		(
         UpperCAmelCase
     )					,		(
         UpperCAmelCase
     )					,		
     )			:				Optional[int]										=					config_and_inputs
     UpperCAmelCase			:				int										=					{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
     return config, inputs_dict






@require_torch
class    __UpperCAmelCase	(			lowerCamelCase__     ,    lowerCamelCase__     ,    unittest.TestCase		):
   UpperCamelCase  	=	False
   UpperCamelCase  	=	False
   UpperCamelCase  	=	(
       (
           FunnelModel,
           FunnelForMaskedLM,
           FunnelForPreTraining,
           FunnelForQuestionAnswering,
           FunnelForTokenClassification,
       )
       if is_torch_available()
       else ()
   )
   UpperCamelCase  	=	(
       {
           """feature-extraction""": (FunnelBaseModel, FunnelModel),
           """fill-mask""": FunnelForMaskedLM,
           """question-answering""": FunnelForQuestionAnswering,
           """text-classification""": FunnelForSequenceClassification,
           """token-classification""": FunnelForTokenClassification,
           """zero-shot""": FunnelForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )



   def 			__magic_name__       (				self				:	Dict,						__A				:	List[Any],						__A				:	str,						__A				:	Dict=False		):
     UpperCAmelCase			:				int										=					super()._prepare_for_class(__A,						__A,						return_labels=__A		)

     if return_labels:
       if model_class in get_values(__A		):
         UpperCAmelCase			:				Tuple										=					torch.zeros(
             (self.model_tester.batch_size, self.model_tester.seq_length),						dtype=torch.long,						device=__A		)
     return inputs_dict



   def 			__magic_name__       (				self				:	List[Any]		):
     UpperCAmelCase			:				Optional[int]										=					FunnelModelTester(self		)
     UpperCAmelCase			:				str										=					ConfigTester(self,						config_class=__A		)



   def 			__magic_name__       (				self				:	Dict		):
     self.config_tester.run_common_tests()



   def 			__magic_name__       (				self				:	int		):
     UpperCAmelCase			:				Optional[Any]										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_model(*__A		)



   def 			__magic_name__       (				self				:	Optional[Any]		):
     UpperCAmelCase			:				List[Any]										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_pretraining(*__A		)



   def 			__magic_name__       (				self				:	Dict		):
     UpperCAmelCase			:				str										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_masked_lm(*__A		)



   def 			__magic_name__       (				self				:	str		):
     UpperCAmelCase			:				Any										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_token_classification(*__A		)



   def 			__magic_name__       (				self				:	List[Any]		):
     UpperCAmelCase			:				int										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_question_answering(*__A		)





   def 			__magic_name__       (				self				:	Any,						__A				:	List[Any]		):
     if hasattr(__A,						'''weight'''		) and module.weight is not None:
       module.weight.data.fill_(3		)
     if hasattr(__A,						'''bias'''		) and module.bias is not None:
       module.bias.data.fill_(3		)

     for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
       if hasattr(__A,						__A		) and getattr(__A,						__A		) is not None:
         UpperCAmelCase			:				Optional[int]										=					getattr(__A,						__A		)
         weight.data.fill_(3		)






@require_torch
class    __UpperCAmelCase	(			lowerCamelCase__     ,    unittest.TestCase		):
   UpperCamelCase  	=	False
   UpperCamelCase  	=	False
   UpperCamelCase  	=	(
       (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
   )



   def 			__magic_name__       (				self				:	List[Any]		):
     UpperCAmelCase			:				Optional[int]										=					FunnelModelTester(self,						base=__A		)
     UpperCAmelCase			:				Tuple										=					ConfigTester(self,						config_class=__A		)



   def 			__magic_name__       (				self				:	Union[str, Any]		):
     self.config_tester.run_common_tests()



   def 			__magic_name__       (				self				:	Any		):
     UpperCAmelCase			:				Union[str, Any]										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_base_model(*__A		)



   def 			__magic_name__       (				self				:	Optional[Any]		):
     UpperCAmelCase			:				Union[str, Any]										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_sequence_classification(*__A		)



   def 			__magic_name__       (				self				:	Optional[int]		):
     UpperCAmelCase			:				Tuple										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_multiple_choice(*__A		)



   def 			__magic_name__       (				self				:	Optional[int]		):
     UpperCAmelCase					,		UpperCAmelCase			:				List[str]										=					self.model_tester.prepare_config_and_inputs_for_common()
     UpperCAmelCase			:				List[str]										=					True

     for model_class in self.all_model_classes:
       if model_class.__name__ == "FunnelBaseModel":
         continue
       UpperCAmelCase			:				Any										=					model_class(__A		)
       model.to(__A		)
       model.train()
       UpperCAmelCase			:				str										=					self._prepare_for_class(__A,						__A,						return_labels=__A		)
       UpperCAmelCase			:				str										=					model(**__A		).loss
       loss.backward()





   def 			__magic_name__       (				self				:	str,						__A				:	Any		):
     if hasattr(__A,						'''weight'''		) and module.weight is not None:
       module.weight.data.fill_(3		)
     if hasattr(__A,						'''bias'''		) and module.bias is not None:
       module.bias.data.fill_(3		)

     for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
       if hasattr(__A,						__A		) and getattr(__A,						__A		) is not None:
         UpperCAmelCase			:				Optional[int]										=					getattr(__A,						__A		)
         weight.data.fill_(3		)






@require_torch
@require_sentencepiece
@require_tokenizers
class    __UpperCAmelCase	(			unittest.TestCase		):



   def 			__magic_name__       (				self				:	int		):
     UpperCAmelCase			:				Optional[Any]										=					1_3
     UpperCAmelCase			:				Tuple										=					7
     UpperCAmelCase			:				Optional[Any]										=					torch.arange(0,						batch_size * sequence_length		).long().reshape(__A,						__A		)
     UpperCAmelCase			:				List[Any]										=					[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
     UpperCAmelCase			:				Optional[Any]										=					torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]		)

     UpperCAmelCase			:				Dict										=					FunnelModel.from_pretrained('''sgugger/funnel-random-tiny'''		)
     UpperCAmelCase			:				List[Any]										=					model(__A,						token_type_ids=__A		)[0].abs()

     UpperCAmelCase			:				Union[str, Any]										=					torch.tensor(2_3_4_4.8_3_5_2		)
     UpperCAmelCase			:				List[Any]										=					torch.tensor(0.8_0_5_2		)
     self.assertTrue(torch.allclose(output.sum(),						__A,						atol=1E-4		)		)
     self.assertTrue(torch.allclose(output.mean(),						__A,						atol=1E-4		)		)

     UpperCAmelCase			:				Union[str, Any]										=					torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]		)
     UpperCAmelCase			:				Union[str, Any]										=					model(__A,						attention_mask=__A,						token_type_ids=__A		)[0].abs()

     UpperCAmelCase			:				str										=					torch.tensor(2_3_4_3.8_4_2_5		)
     UpperCAmelCase			:				Optional[Any]										=					torch.tensor(0.8_0_4_9		)
     self.assertTrue(torch.allclose(output.sum(),						__A,						atol=1E-4		)		)
     self.assertTrue(torch.allclose(output.mean(),						__A,						atol=1E-4		)		)





   @slow
   def 			__magic_name__       (				self				:	int		):
     UpperCAmelCase			:				List[Any]										=					FunnelTokenizer.from_pretrained('''huggingface/funnel-small'''		)
     UpperCAmelCase			:				List[str]										=					FunnelModel.from_pretrained('''huggingface/funnel-small'''		)
     UpperCAmelCase			:				List[str]										=					tokenizer('''Hello! I am the Funnel Transformer model.''',						return_tensors='''pt'''		)
     UpperCAmelCase			:				Optional[Any]										=					model(**__A		)[0]

     UpperCAmelCase			:				int										=					torch.tensor(2_3_5.7_2_4_6		)
     UpperCAmelCase			:				Optional[int]										=					torch.tensor(0.0_2_5_6		)
     self.assertTrue(torch.allclose(output.sum(),						__A,						atol=1E-4		)		)
     self.assertTrue(torch.allclose(output.mean(),						__A,						atol=1E-4		)		)





