





import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    MODEL_FOR_PRETRAINING_MAPPING,
				    FunnelBaseModel,
				    FunnelForMaskedLM,
				    FunnelForMultipleChoice,
				    FunnelForPreTraining,
				    FunnelForQuestionAnswering,
				    FunnelForSequenceClassification,
				    FunnelForTokenClassification,
				    FunnelModel,
				)
class 							snake_case_		:







		def __init__(		self      ,  __lowercase      ,  __lowercase=1_3      ,  __lowercase=7      ,  __lowercase=True      ,  __lowercase=True      ,  __lowercase=True      ,  __lowercase=True      ,  __lowercase=9_9      ,  __lowercase=[1, 1, 2]      ,  __lowercase=1      ,  __lowercase=3_2      ,  __lowercase=4      ,  __lowercase=8      ,  __lowercase=3_7      ,  __lowercase="gelu_new"      ,  __lowercase=0.1      ,  __lowercase=0.1      ,  __lowercase=0.0      ,  __lowercase=5_1_2      ,  __lowercase=3      ,  __lowercase=0.0_2      ,  __lowercase=3      ,  __lowercase=4      ,  __lowercase=None      ,  __lowercase=False      ,  )					-> Union[str, Any]:
				lowerCamelCase :			List[str]    		=parent
				lowerCamelCase :			str    		=batch_size
				lowerCamelCase :			str    		=seq_length
				lowerCamelCase :			Dict    		=is_training
				lowerCamelCase :			Union[str, Any]    		=use_input_mask
				lowerCamelCase :			str    		=use_token_type_ids
				lowerCamelCase :			Union[str, Any]    		=use_labels
				lowerCamelCase :			Tuple    		=vocab_size
				lowerCamelCase :			List[Any]    		=block_sizes
				lowerCamelCase :			Dict    		=num_decoder_layers
				lowerCamelCase :			Union[str, Any]    		=d_model
				lowerCamelCase :			Tuple    		=n_head
				lowerCamelCase :			Dict    		=d_head
				lowerCamelCase :			Union[str, Any]    		=d_inner
				lowerCamelCase :			Optional[Any]    		=hidden_act
				lowerCamelCase :			Union[str, Any]    		=hidden_dropout
				lowerCamelCase :			Dict    		=attention_dropout
				lowerCamelCase :			str    		=activation_dropout
				lowerCamelCase :			Tuple    		=max_position_embeddings
				lowerCamelCase :			Optional[int]    		=type_vocab_size
				lowerCamelCase :			List[Any]    		=2
				lowerCamelCase :			Tuple    		=num_labels
				lowerCamelCase :			Union[str, Any]    		=num_choices
				lowerCamelCase :			Optional[int]    		=scope
				lowerCamelCase :			str    		=initializer_std

				# Used in the tests to check the size of the first attention layer
				lowerCamelCase :			str    		=n_head
				# Used in the tests to check the size of the first hidden state
				lowerCamelCase :			Any    		=self.d_model
				# Used in the tests to check the number of output hidden states/attentions
				lowerCamelCase :			int    		=sum(self.block_sizes ) + (0 if base else self.num_decoder_layers)
				# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
				# the last hidden state of the first block (which is the first hidden state of the decoder).
				if not base:
						lowerCamelCase :			Dict    		=self.num_hidden_layers + 2







		def       __lowercase (		self )					-> Optional[Any]:
				lowerCamelCase :			List[Any]    		=ids_tensor([self.batch_size, self.seq_length]      ,  self.vocab_size )

				lowerCamelCase :			Dict    		=None
				if self.use_input_mask:
						lowerCamelCase :			Tuple    		=ids_tensor([self.batch_size, self.seq_length]      ,  vocab_size=2 )

				lowerCamelCase :			Optional[int]    		=None
				if self.use_token_type_ids:
						lowerCamelCase :			List[str]    		=ids_tensor([self.batch_size, self.seq_length]      ,  self.type_vocab_size )

				lowerCamelCase :			Optional[Any]    		=None
				lowerCamelCase :			int    		=None
				lowerCamelCase :			Union[str, Any]    		=None
				if self.use_labels:
						lowerCamelCase :			Any    		=ids_tensor([self.batch_size]      ,  self.type_sequence_label_size )
						lowerCamelCase :			Any    		=ids_tensor([self.batch_size, self.seq_length]      ,  self.num_labels )
						lowerCamelCase :			List[str]    		=ids_tensor([self.batch_size]      ,  self.num_choices )
						lowerCamelCase :			List[Any]    		=ids_tensor([self.batch_size, self.seq_length]      ,  1 )

				lowerCamelCase :			List[str]    		=self.get_config()

				return (
				    config,
				    input_ids,
				    token_type_ids,
				    input_mask,
				    sequence_labels,
				    token_labels,
				    choice_labels,
				    fake_token_labels,
				)







		def       __lowercase (		self )					-> Any:
				return FunnelConfig(
				    vocab_size=self.vocab_size      ,  block_sizes=self.block_sizes      ,  num_decoder_layers=self.num_decoder_layers      ,  d_model=self.d_model      ,  n_head=self.n_head      ,  d_head=self.d_head      ,  d_inner=self.d_inner      ,  hidden_act=self.hidden_act      ,  hidden_dropout=self.hidden_dropout      ,  attention_dropout=self.attention_dropout      ,  activation_dropout=self.activation_dropout      ,  max_position_embeddings=self.max_position_embeddings      ,  type_vocab_size=self.type_vocab_size      ,  initializer_std=self.initializer_std      ,  )







		def       __lowercase (		self      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  )					-> str:
				lowerCamelCase :			Tuple    		=FunnelModel(config=__lowercase )
				model.to(__lowercase )
				model.eval()
				lowerCamelCase :			Optional[Any]    		=model(__lowercase      ,  attention_mask=__lowercase      ,  token_type_ids=__lowercase )
				lowerCamelCase :			str    		=model(__lowercase      ,  token_type_ids=__lowercase )
				lowerCamelCase :			Union[str, Any]    		=model(__lowercase )
				self.parent.assertEqual(result.last_hidden_state.shape      ,  (self.batch_size, self.seq_length, self.d_model) )

				lowerCamelCase :			Union[str, Any]    		=False
				lowerCamelCase :			Optional[Any]    		=model(__lowercase )
				self.parent.assertEqual(result.last_hidden_state.shape      ,  (self.batch_size, self.seq_length, self.d_model) )

				lowerCamelCase :			str    		=False
				lowerCamelCase :			List[str]    		=model(__lowercase )
				self.parent.assertEqual(result.last_hidden_state.shape      ,  (self.batch_size, self.seq_length, self.d_model) )







		def       __lowercase (		self      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  )					-> Dict:
				lowerCamelCase :			Optional[Any]    		=FunnelBaseModel(config=__lowercase )
				model.to(__lowercase )
				model.eval()
				lowerCamelCase :			int    		=model(__lowercase      ,  attention_mask=__lowercase      ,  token_type_ids=__lowercase )
				lowerCamelCase :			List[str]    		=model(__lowercase      ,  token_type_ids=__lowercase )
				lowerCamelCase :			Union[str, Any]    		=model(__lowercase )
				self.parent.assertEqual(result.last_hidden_state.shape      ,  (self.batch_size, 2, self.d_model) )

				lowerCamelCase :			Optional[Any]    		=False
				lowerCamelCase :			Tuple    		=model(__lowercase )
				self.parent.assertEqual(result.last_hidden_state.shape      ,  (self.batch_size, 3, self.d_model) )

				lowerCamelCase :			Optional[int]    		=False
				lowerCamelCase :			List[str]    		=model(__lowercase )
				self.parent.assertEqual(result.last_hidden_state.shape      ,  (self.batch_size, 2, self.d_model) )







		def       __lowercase (		self      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  )					-> str:
				lowerCamelCase :			List[str]    		=self.num_labels
				lowerCamelCase :			List[str]    		=FunnelForPreTraining(config=__lowercase )
				model.to(__lowercase )
				model.eval()
				lowerCamelCase :			Union[str, Any]    		=model(__lowercase      ,  attention_mask=__lowercase      ,  token_type_ids=__lowercase      ,  labels=__lowercase )
				self.parent.assertEqual(result.logits.shape      ,  (self.batch_size, self.seq_length) )







		def       __lowercase (		self      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  )					-> Dict:
				lowerCamelCase :			Any    		=FunnelForMaskedLM(config=__lowercase )
				model.to(__lowercase )
				model.eval()
				lowerCamelCase :			Optional[int]    		=model(__lowercase      ,  attention_mask=__lowercase      ,  token_type_ids=__lowercase      ,  labels=__lowercase )
				self.parent.assertEqual(result.logits.shape      ,  (self.batch_size, self.seq_length, self.vocab_size) )







		def       __lowercase (		self      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  )					-> Union[str, Any]:
				lowerCamelCase :			str    		=self.num_labels
				lowerCamelCase :			str    		=FunnelForSequenceClassification(__lowercase )
				model.to(__lowercase )
				model.eval()
				lowerCamelCase :			Any    		=model(__lowercase      ,  attention_mask=__lowercase      ,  token_type_ids=__lowercase      ,  labels=__lowercase )
				self.parent.assertEqual(result.logits.shape      ,  (self.batch_size, self.num_labels) )







		def       __lowercase (		self      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  )					-> Union[str, Any]:
				lowerCamelCase :			List[Any]    		=self.num_choices
				lowerCamelCase :			List[Any]    		=FunnelForMultipleChoice(config=__lowercase )
				model.to(__lowercase )
				model.eval()
				lowerCamelCase :			int    		=input_ids.unsqueeze(1 ).expand(-1      ,  self.num_choices      ,  -1 ).contiguous()
				lowerCamelCase :			Optional[Any]    		=token_type_ids.unsqueeze(1 ).expand(-1      ,  self.num_choices      ,  -1 ).contiguous()
				lowerCamelCase :			List[str]    		=input_mask.unsqueeze(1 ).expand(-1      ,  self.num_choices      ,  -1 ).contiguous()
				lowerCamelCase :			Any    		=model(
				    __lowercase      ,  attention_mask=__lowercase      ,  token_type_ids=__lowercase      ,  labels=__lowercase      ,  )
				self.parent.assertEqual(result.logits.shape      ,  (self.batch_size, self.num_choices) )







		def       __lowercase (		self      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  )					-> List[str]:
				lowerCamelCase :			List[str]    		=self.num_labels
				lowerCamelCase :			Tuple    		=FunnelForTokenClassification(config=__lowercase )
				model.to(__lowercase )
				model.eval()
				lowerCamelCase :			List[Any]    		=model(__lowercase      ,  attention_mask=__lowercase      ,  token_type_ids=__lowercase      ,  labels=__lowercase )
				self.parent.assertEqual(result.logits.shape      ,  (self.batch_size, self.seq_length, self.num_labels) )







		def       __lowercase (		self      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  )					-> Optional[int]:
				lowerCamelCase :			str    		=FunnelForQuestionAnswering(config=__lowercase )
				model.to(__lowercase )
				model.eval()
				lowerCamelCase :			Dict    		=model(
				    __lowercase      ,  attention_mask=__lowercase      ,  token_type_ids=__lowercase      ,  start_positions=__lowercase      ,  end_positions=__lowercase      ,  )
				self.parent.assertEqual(result.start_logits.shape      ,  (self.batch_size, self.seq_length) )
				self.parent.assertEqual(result.end_logits.shape      ,  (self.batch_size, self.seq_length) )







		def       __lowercase (		self )					-> Union[str, Any]:
				lowerCamelCase :			Any    		=self.prepare_config_and_inputs()
				(
				    (
				    lowerCamelCase
				)		, (
				    lowerCamelCase
				)		, (
				    lowerCamelCase
				)		, (
				    lowerCamelCase
				)		, (
				    lowerCamelCase
				)		, (
				    lowerCamelCase
				)		, (
				    lowerCamelCase
				)		, (
				    lowerCamelCase
				)		, 
				) :			List[Any]    		=config_and_inputs
				lowerCamelCase :			Tuple    		={'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
				return config, inputs_dict

@require_torch
class 							snake_case_		(      _A    ,       _A    ,       unittest.TestCase):
		lowerCamelCase     :Optional[Any]							=  False
		lowerCamelCase     :Optional[int]							=  False
		lowerCamelCase     :Dict							=  (
		    (
		        FunnelModel,
		        FunnelForMaskedLM,
		        FunnelForPreTraining,
		        FunnelForQuestionAnswering,
		        FunnelForTokenClassification,
		    )
		    if is_torch_available()
		    else ()
		)
		lowerCamelCase     :Optional[Any]							=  (
		    {
		        "feature-extraction": (FunnelBaseModel, FunnelModel),
		        "fill-mask": FunnelForMaskedLM,
		        "question-answering": FunnelForQuestionAnswering,
		        "text-classification": FunnelForSequenceClassification,
		        "token-classification": FunnelForTokenClassification,
		        "zero-shot": FunnelForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)







		def       __lowercase (		self      ,  __lowercase      ,  __lowercase      ,  __lowercase=False )					-> Union[str, Any]:
				lowerCamelCase :			Optional[Any]    		=super()._prepare_for_class(__lowercase      ,  __lowercase      ,  return_labels=__lowercase )

				if return_labels:
						if model_class in get_values(__lowercase ):
								lowerCamelCase :			Dict    		=torch.zeros(
								    (self.model_tester.batch_size, self.model_tester.seq_length)      ,  dtype=torch.long      ,  device=__lowercase )
				return inputs_dict







		def       __lowercase (		self )					-> int:
				lowerCamelCase :			Optional[Any]    		=FunnelModelTester(self )
				lowerCamelCase :			List[str]    		=ConfigTester(self      ,  config_class=__lowercase )







		def       __lowercase (		self )					-> Tuple:
				self.config_tester.run_common_tests()







		def       __lowercase (		self )					-> List[str]:
				lowerCamelCase :			Union[str, Any]    		=self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_model(*__lowercase )







		def       __lowercase (		self )					-> Optional[int]:
				lowerCamelCase :			List[Any]    		=self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_pretraining(*__lowercase )







		def       __lowercase (		self )					-> Dict:
				lowerCamelCase :			str    		=self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_masked_lm(*__lowercase )







		def       __lowercase (		self )					-> int:
				lowerCamelCase :			int    		=self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_token_classification(*__lowercase )







		def       __lowercase (		self )					-> List[str]:
				lowerCamelCase :			List[Any]    		=self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_question_answering(*__lowercase )







		def       __lowercase (		self      ,  __lowercase )					-> Union[str, Any]:
				if hasattr(__lowercase      ,  '''weight''' ) and module.weight is not None:
						module.weight.data.fill_(3 )
				if hasattr(__lowercase      ,  '''bias''' ) and module.bias is not None:
						module.bias.data.fill_(3 )

				for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
						if hasattr(__lowercase      ,  __lowercase ) and getattr(__lowercase      ,  __lowercase ) is not None:
								lowerCamelCase :			Tuple    		=getattr(__lowercase      ,  __lowercase )
								weight.data.fill_(3 )

@require_torch
class 							snake_case_		(      _A    ,       unittest.TestCase):
		lowerCamelCase     :Tuple							=  False
		lowerCamelCase     :List[str]							=  False
		lowerCamelCase     :List[Any]							=  (
		    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
		)







		def       __lowercase (		self )					-> str:
				lowerCamelCase :			str    		=FunnelModelTester(self      ,  base=__lowercase )
				lowerCamelCase :			Dict    		=ConfigTester(self      ,  config_class=__lowercase )







		def       __lowercase (		self )					-> int:
				self.config_tester.run_common_tests()







		def       __lowercase (		self )					-> Optional[Any]:
				lowerCamelCase :			Tuple    		=self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_base_model(*__lowercase )







		def       __lowercase (		self )					-> int:
				lowerCamelCase :			Any    		=self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_sequence_classification(*__lowercase )







		def       __lowercase (		self )					-> int:
				lowerCamelCase :			Any    		=self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_multiple_choice(*__lowercase )







		def       __lowercase (		self )					-> Optional[Any]:
				lowerCamelCase		, lowerCamelCase :			List[str]    		=self.model_tester.prepare_config_and_inputs_for_common()
				lowerCamelCase :			str    		=True

				for model_class in self.all_model_classes:
						if model_class.__name__ == "FunnelBaseModel":
								continue
						lowerCamelCase :			Dict    		=model_class(__lowercase )
						model.to(__lowercase )
						model.train()
						lowerCamelCase :			str    		=self._prepare_for_class(__lowercase      ,  __lowercase      ,  return_labels=__lowercase )
						lowerCamelCase :			Optional[Any]    		=model(**__lowercase ).loss
						loss.backward()







		def       __lowercase (		self      ,  __lowercase )					-> int:
				if hasattr(__lowercase      ,  '''weight''' ) and module.weight is not None:
						module.weight.data.fill_(3 )
				if hasattr(__lowercase      ,  '''bias''' ) and module.bias is not None:
						module.bias.data.fill_(3 )

				for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
						if hasattr(__lowercase      ,  __lowercase ) and getattr(__lowercase      ,  __lowercase ) is not None:
								lowerCamelCase :			Dict    		=getattr(__lowercase      ,  __lowercase )
								weight.data.fill_(3 )

@require_torch
@require_sentencepiece
@require_tokenizers
class 							snake_case_		(      unittest.TestCase):







		def       __lowercase (		self )					-> Union[str, Any]:
				lowerCamelCase :			Tuple    		=1_3
				lowerCamelCase :			List[Any]    		=7
				lowerCamelCase :			Optional[int]    		=torch.arange(0      ,  batch_size * sequence_length ).long().reshape(__lowercase      ,  __lowercase )
				lowerCamelCase :			Optional[int]    		=[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
				lowerCamelCase :			str    		=torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths] )

				lowerCamelCase :			int    		=FunnelModel.from_pretrained('''sgugger/funnel-random-tiny''' )
				lowerCamelCase :			Any    		=model(__lowercase      ,  token_type_ids=__lowercase )[0].abs()

				lowerCamelCase :			Any    		=torch.tensor(2_3_4_4.8_3_5_2 )
				lowerCamelCase :			Tuple    		=torch.tensor(0.8_0_5_2 )
				self.assertTrue(torch.allclose(output.sum()      ,  __lowercase      ,  atol=1e-4 ) )
				self.assertTrue(torch.allclose(output.mean()      ,  __lowercase      ,  atol=1e-4 ) )

				lowerCamelCase :			str    		=torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]] )
				lowerCamelCase :			Union[str, Any]    		=model(__lowercase      ,  attention_mask=__lowercase      ,  token_type_ids=__lowercase )[0].abs()

				lowerCamelCase :			Optional[Any]    		=torch.tensor(2_3_4_3.8_4_2_5 )
				lowerCamelCase :			Union[str, Any]    		=torch.tensor(0.8_0_4_9 )
				self.assertTrue(torch.allclose(output.sum()      ,  __lowercase      ,  atol=1e-4 ) )
				self.assertTrue(torch.allclose(output.mean()      ,  __lowercase      ,  atol=1e-4 ) )







		@slow
		def       __lowercase (		self )					-> List[Any]:
				lowerCamelCase :			Any    		=FunnelTokenizer.from_pretrained('''huggingface/funnel-small''' )
				lowerCamelCase :			Any    		=FunnelModel.from_pretrained('''huggingface/funnel-small''' )
				lowerCamelCase :			List[str]    		=tokenizer('''Hello! I am the Funnel Transformer model.'''      ,  return_tensors='''pt''' )
				lowerCamelCase :			Optional[Any]    		=model(**__lowercase )[0]

				lowerCamelCase :			Union[str, Any]    		=torch.tensor(2_3_5.7_2_4_6 )
				lowerCamelCase :			Dict    		=torch.tensor(0.0_2_5_6 )
				self.assertTrue(torch.allclose(output.sum()      ,  __lowercase      ,  atol=1e-4 ) )
				self.assertTrue(torch.allclose(output.mean()      ,  __lowercase      ,  atol=1e-4 ) )








def        A__						(							SCREAMING_SNAKE_CASE_ = 1_0_0_0		) ->  int:
		lowerCamelCase :			Dict    		=[]
		lowerCamelCase :			Dict    		=[]
		lowerCamelCase :			Union[str, Any]    		=3
		lowerCamelCase :			List[str]    		=5
		lowerCamelCase :			Union[str, Any]    		=1
		while True:
				lowerCamelCase :			Dict    		=z * temp
				if result < n:
						zmulti.append(SCREAMING_SNAKE_CASE_		)
						temp += 1
				else:
						lowerCamelCase :			List[Any]    		=1
						break
		while True:
				lowerCamelCase :			Optional[Any]    		=x * temp
				if result < n:
						xmulti.append(SCREAMING_SNAKE_CASE_		)
						temp += 1
				else:
						break
		lowerCamelCase :			str    		=list(set(xmulti + zmulti		)		)
		return sum(SCREAMING_SNAKE_CASE_		)


if __name__ == "__main__":
				print(F"""{solution() = }""")

