
import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MODEL_FOR_PRETRAINING_MAPPING,
							    FunnelBaseModel,
							    FunnelForMaskedLM,
							    FunnelForMultipleChoice,
							    FunnelForPreTraining,
							    FunnelForQuestionAnswering,
							    FunnelForSequenceClassification,
							    FunnelForTokenClassification,
							    FunnelModel,
							)
class       A__  :





							def __init__(					self    :  List[Any]	,					_UpperCAmelCase    :  Tuple	,					_UpperCAmelCase    :  Optional[int]=13	,					_UpperCAmelCase    :  List[str]=7	,					_UpperCAmelCase    :  Optional[int]=True	,					_UpperCAmelCase    :  Any=True	,					_UpperCAmelCase    :  List[Any]=True	,					_UpperCAmelCase    :  Dict=True	,					_UpperCAmelCase    :  str=99	,					_UpperCAmelCase    :  Optional[Any]=[1, 1, 2]	,					_UpperCAmelCase    :  Any=1	,					_UpperCAmelCase    :  Any=32	,					_UpperCAmelCase    :  Tuple=4	,					_UpperCAmelCase    :  int=8	,					_UpperCAmelCase    :  Optional[Any]=37	,					_UpperCAmelCase    :  Optional[Any]="gelu_new"	,					_UpperCAmelCase    :  Optional[int]=0.1	,					_UpperCAmelCase    :  str=0.1	,					_UpperCAmelCase    :  List[str]=0.0	,					_UpperCAmelCase    :  Tuple=5_12	,					_UpperCAmelCase    :  Optional[Any]=3	,					_UpperCAmelCase    :  Optional[Any]=0.02	,					_UpperCAmelCase    :  Optional[Any]=3	,					_UpperCAmelCase    :  List[str]=4	,					_UpperCAmelCase    :  Any=None	,					_UpperCAmelCase    :  List[str]=False	,					)		->						str:



											"""simple docstring"""


											__lowercase								=   parent
											__lowercase								=   batch_size
											__lowercase								=   seq_length
											__lowercase								=   is_training
											__lowercase								=   use_input_mask
											__lowercase								=   use_token_type_ids
											__lowercase								=   use_labels
											__lowercase								=   vocab_size
											__lowercase								=   block_sizes
											__lowercase								=   num_decoder_layers
											__lowercase								=   d_model
											__lowercase								=   n_head
											__lowercase								=   d_head
											__lowercase								=   d_inner
											__lowercase								=   hidden_act
											__lowercase								=   hidden_dropout
											__lowercase								=   attention_dropout
											__lowercase								=   activation_dropout
											__lowercase								=   max_position_embeddings
											__lowercase								=   type_vocab_size
											__lowercase								=   2
											__lowercase								=   num_labels
											__lowercase								=   num_choices
											__lowercase								=   scope
											__lowercase								=   initializer_std

											# Used in the tests to check the size of the first attention layer
											__lowercase								=   n_head
											# Used in the tests to check the size of the first hidden state
											__lowercase								=   self.d_model
											# Used in the tests to check the number of output hidden states/attentions
											__lowercase								=   sum(self.block_sizes							) + (0 if base else self.num_decoder_layers)
											# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
											# the last hidden state of the first block (which is the first hidden state of the decoder).
											if not base:
															__lowercase								=   self.num_hidden_layers + 2





							def      a__					(					self    :  Tuple							)		->						Optional[int]:



											"""simple docstring"""


											__lowercase								=   ids_tensor([self.batch_size, self.seq_length]	,					self.vocab_size							)

											__lowercase								=   None
											if self.use_input_mask:
															__lowercase								=   ids_tensor([self.batch_size, self.seq_length]	,					vocab_size=2							)

											__lowercase								=   None
											if self.use_token_type_ids:
															__lowercase								=   ids_tensor([self.batch_size, self.seq_length]	,					self.type_vocab_size							)

											__lowercase								=   None
											__lowercase								=   None
											__lowercase								=   None
											if self.use_labels:
															__lowercase								=   ids_tensor([self.batch_size]	,					self.type_sequence_label_size							)
															__lowercase								=   ids_tensor([self.batch_size, self.seq_length]	,					self.num_labels							)
															__lowercase								=   ids_tensor([self.batch_size]	,					self.num_choices							)
															__lowercase								=   ids_tensor([self.batch_size, self.seq_length]	,					1							)

											__lowercase								=   self.get_config()

											return (
											    config,
											    input_ids,
											    token_type_ids,
											    input_mask,
											    sequence_labels,
											    token_labels,
											    choice_labels,
											    fake_token_labels,
											)





							def      a__					(					self    :  Union[str, Any]							)		->						Dict:



											"""simple docstring"""


											return FunnelConfig(
											    vocab_size=self.vocab_size	,					block_sizes=self.block_sizes	,					num_decoder_layers=self.num_decoder_layers	,					d_model=self.d_model	,					n_head=self.n_head	,					d_head=self.d_head	,					d_inner=self.d_inner	,					hidden_act=self.hidden_act	,					hidden_dropout=self.hidden_dropout	,					attention_dropout=self.attention_dropout	,					activation_dropout=self.activation_dropout	,					max_position_embeddings=self.max_position_embeddings	,					type_vocab_size=self.type_vocab_size	,					initializer_std=self.initializer_std	,					)





							def      a__					(					self    :  List[str]	,					_UpperCAmelCase    :  Tuple	,					_UpperCAmelCase    :  str	,					_UpperCAmelCase    :  str	,					_UpperCAmelCase    :  Dict	,					_UpperCAmelCase    :  Any	,					_UpperCAmelCase    :  List[Any]	,					_UpperCAmelCase    :  List[Any]	,					_UpperCAmelCase    :  Optional[Any]	,					)		->						Tuple:



											"""simple docstring"""


											__lowercase								=   FunnelModel(config=lowerCAmelCase__							)
											model.to(lowerCAmelCase__							)
											model.eval()
											__lowercase								=   model(lowerCAmelCase__	,					attention_mask=lowerCAmelCase__	,					token_type_ids=lowerCAmelCase__							)
											__lowercase								=   model(lowerCAmelCase__	,					token_type_ids=lowerCAmelCase__							)
											__lowercase								=   model(lowerCAmelCase__							)
											self.parent.assertEqual(result.last_hidden_state.shape	,					(self.batch_size, self.seq_length, self.d_model)							)

											__lowercase								=   False
											__lowercase								=   model(lowerCAmelCase__							)
											self.parent.assertEqual(result.last_hidden_state.shape	,					(self.batch_size, self.seq_length, self.d_model)							)

											__lowercase								=   False
											__lowercase								=   model(lowerCAmelCase__							)
											self.parent.assertEqual(result.last_hidden_state.shape	,					(self.batch_size, self.seq_length, self.d_model)							)





							def      a__					(					self    :  Optional[Any]	,					_UpperCAmelCase    :  Optional[Any]	,					_UpperCAmelCase    :  Tuple	,					_UpperCAmelCase    :  str	,					_UpperCAmelCase    :  int	,					_UpperCAmelCase    :  Any	,					_UpperCAmelCase    :  Union[str, Any]	,					_UpperCAmelCase    :  str	,					_UpperCAmelCase    :  Dict	,					)		->						Any:



											"""simple docstring"""


											__lowercase								=   FunnelBaseModel(config=lowerCAmelCase__							)
											model.to(lowerCAmelCase__							)
											model.eval()
											__lowercase								=   model(lowerCAmelCase__	,					attention_mask=lowerCAmelCase__	,					token_type_ids=lowerCAmelCase__							)
											__lowercase								=   model(lowerCAmelCase__	,					token_type_ids=lowerCAmelCase__							)
											__lowercase								=   model(lowerCAmelCase__							)
											self.parent.assertEqual(result.last_hidden_state.shape	,					(self.batch_size, 2, self.d_model)							)

											__lowercase								=   False
											__lowercase								=   model(lowerCAmelCase__							)
											self.parent.assertEqual(result.last_hidden_state.shape	,					(self.batch_size, 3, self.d_model)							)

											__lowercase								=   False
											__lowercase								=   model(lowerCAmelCase__							)
											self.parent.assertEqual(result.last_hidden_state.shape	,					(self.batch_size, 2, self.d_model)							)





							def      a__					(					self    :  Optional[Any]	,					_UpperCAmelCase    :  Tuple	,					_UpperCAmelCase    :  Optional[int]	,					_UpperCAmelCase    :  List[str]	,					_UpperCAmelCase    :  Any	,					_UpperCAmelCase    :  List[str]	,					_UpperCAmelCase    :  Optional[int]	,					_UpperCAmelCase    :  List[str]	,					_UpperCAmelCase    :  Union[str, Any]	,					)		->						List[Any]:



											"""simple docstring"""


											__lowercase								=   self.num_labels
											__lowercase								=   FunnelForPreTraining(config=lowerCAmelCase__							)
											model.to(lowerCAmelCase__							)
											model.eval()
											__lowercase								=   model(lowerCAmelCase__	,					attention_mask=lowerCAmelCase__	,					token_type_ids=lowerCAmelCase__	,					labels=lowerCAmelCase__							)
											self.parent.assertEqual(result.logits.shape	,					(self.batch_size, self.seq_length)							)





							def      a__					(					self    :  Dict	,					_UpperCAmelCase    :  str	,					_UpperCAmelCase    :  List[str]	,					_UpperCAmelCase    :  List[str]	,					_UpperCAmelCase    :  int	,					_UpperCAmelCase    :  List[str]	,					_UpperCAmelCase    :  Dict	,					_UpperCAmelCase    :  int	,					_UpperCAmelCase    :  Union[str, Any]	,					)		->						str:



											"""simple docstring"""


											__lowercase								=   FunnelForMaskedLM(config=lowerCAmelCase__							)
											model.to(lowerCAmelCase__							)
											model.eval()
											__lowercase								=   model(lowerCAmelCase__	,					attention_mask=lowerCAmelCase__	,					token_type_ids=lowerCAmelCase__	,					labels=lowerCAmelCase__							)
											self.parent.assertEqual(result.logits.shape	,					(self.batch_size, self.seq_length, self.vocab_size)							)





							def      a__					(					self    :  Dict	,					_UpperCAmelCase    :  Optional[Any]	,					_UpperCAmelCase    :  int	,					_UpperCAmelCase    :  str	,					_UpperCAmelCase    :  List[str]	,					_UpperCAmelCase    :  Tuple	,					_UpperCAmelCase    :  Optional[int]	,					_UpperCAmelCase    :  Optional[Any]	,					_UpperCAmelCase    :  List[Any]	,					)		->						Optional[int]:



											"""simple docstring"""


											__lowercase								=   self.num_labels
											__lowercase								=   FunnelForSequenceClassification(lowerCAmelCase__							)
											model.to(lowerCAmelCase__							)
											model.eval()
											__lowercase								=   model(lowerCAmelCase__	,					attention_mask=lowerCAmelCase__	,					token_type_ids=lowerCAmelCase__	,					labels=lowerCAmelCase__							)
											self.parent.assertEqual(result.logits.shape	,					(self.batch_size, self.num_labels)							)





							def      a__					(					self    :  List[Any]	,					_UpperCAmelCase    :  str	,					_UpperCAmelCase    :  List[Any]	,					_UpperCAmelCase    :  List[str]	,					_UpperCAmelCase    :  Union[str, Any]	,					_UpperCAmelCase    :  Any	,					_UpperCAmelCase    :  Tuple	,					_UpperCAmelCase    :  str	,					_UpperCAmelCase    :  Optional[Any]	,					)		->						Optional[int]:



											"""simple docstring"""


											__lowercase								=   self.num_choices
											__lowercase								=   FunnelForMultipleChoice(config=lowerCAmelCase__							)
											model.to(lowerCAmelCase__							)
											model.eval()
											__lowercase								=   input_ids.unsqueeze(1							).expand(-1	,					self.num_choices	,					-1							).contiguous()
											__lowercase								=   token_type_ids.unsqueeze(1							).expand(-1	,					self.num_choices	,					-1							).contiguous()
											__lowercase								=   input_mask.unsqueeze(1							).expand(-1	,					self.num_choices	,					-1							).contiguous()
											__lowercase								=   model(
											    lowerCAmelCase__	,					attention_mask=lowerCAmelCase__	,					token_type_ids=lowerCAmelCase__	,					labels=lowerCAmelCase__	,					)
											self.parent.assertEqual(result.logits.shape	,					(self.batch_size, self.num_choices)							)





							def      a__					(					self    :  Dict	,					_UpperCAmelCase    :  List[str]	,					_UpperCAmelCase    :  int	,					_UpperCAmelCase    :  List[str]	,					_UpperCAmelCase    :  int	,					_UpperCAmelCase    :  str	,					_UpperCAmelCase    :  Union[str, Any]	,					_UpperCAmelCase    :  Optional[Any]	,					_UpperCAmelCase    :  List[Any]	,					)		->						Tuple:



											"""simple docstring"""


											__lowercase								=   self.num_labels
											__lowercase								=   FunnelForTokenClassification(config=lowerCAmelCase__							)
											model.to(lowerCAmelCase__							)
											model.eval()
											__lowercase								=   model(lowerCAmelCase__	,					attention_mask=lowerCAmelCase__	,					token_type_ids=lowerCAmelCase__	,					labels=lowerCAmelCase__							)
											self.parent.assertEqual(result.logits.shape	,					(self.batch_size, self.seq_length, self.num_labels)							)





							def      a__					(					self    :  Any	,					_UpperCAmelCase    :  Optional[int]	,					_UpperCAmelCase    :  int	,					_UpperCAmelCase    :  Optional[Any]	,					_UpperCAmelCase    :  List[Any]	,					_UpperCAmelCase    :  Optional[Any]	,					_UpperCAmelCase    :  List[str]	,					_UpperCAmelCase    :  Tuple	,					_UpperCAmelCase    :  Any	,					)		->						int:



											"""simple docstring"""


											__lowercase								=   FunnelForQuestionAnswering(config=lowerCAmelCase__							)
											model.to(lowerCAmelCase__							)
											model.eval()
											__lowercase								=   model(
											    lowerCAmelCase__	,					attention_mask=lowerCAmelCase__	,					token_type_ids=lowerCAmelCase__	,					start_positions=lowerCAmelCase__	,					end_positions=lowerCAmelCase__	,					)
											self.parent.assertEqual(result.start_logits.shape	,					(self.batch_size, self.seq_length)							)
											self.parent.assertEqual(result.end_logits.shape	,					(self.batch_size, self.seq_length)							)





							def      a__					(					self    :  str							)		->						Optional[int]:



											"""simple docstring"""


											__lowercase								=   self.prepare_config_and_inputs()
											(
											    (
											    __lowercase      
											)						,   (
											    __lowercase      
											)						,   (
											    __lowercase      
											)						,   (
											    __lowercase      
											)						,   (
											    __lowercase      
											)						,   (
											    __lowercase      
											)						,   (
											    __lowercase      
											)						,   (
											    __lowercase      
											)						,         
											)								=   config_and_inputs
											__lowercase								=   {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
											return config, inputs_dict






@require_torch
class       A__  (		lowerCAmelCase__       ,		lowerCAmelCase__       ,		unittest.TestCase							):
							lowerCAmelCase__						:		Any            =				False
							lowerCAmelCase__						:		Union[str, Any]            =				False
							lowerCAmelCase__						:		Union[str, Any]            =				(
							    (
							        FunnelModel,
							        FunnelForMaskedLM,
							        FunnelForPreTraining,
							        FunnelForQuestionAnswering,
							        FunnelForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							lowerCAmelCase__						:		Any            =				(
							    {
							        "feature-extraction": (FunnelBaseModel, FunnelModel),
							        "fill-mask": FunnelForMaskedLM,
							        "question-answering": FunnelForQuestionAnswering,
							        "text-classification": FunnelForSequenceClassification,
							        "token-classification": FunnelForTokenClassification,
							        "zero-shot": FunnelForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)





							def      a__					(					self    :  List[Any]	,					_UpperCAmelCase    :  str	,					_UpperCAmelCase    :  str	,					_UpperCAmelCase    :  str=False							)		->						List[Any]:



											"""simple docstring"""


											__lowercase								=   super()._prepare_for_class(lowerCAmelCase__	,					lowerCAmelCase__	,					return_labels=lowerCAmelCase__							)

											if return_labels:
															if model_class in get_values(lowerCAmelCase__							):
																			__lowercase								=   torch.zeros(
																			    (self.model_tester.batch_size, self.model_tester.seq_length)	,					dtype=torch.long	,					device=lowerCAmelCase__							)
											return inputs_dict





							def      a__					(					self    :  Tuple							)		->						List[Any]:



											"""simple docstring"""


											__lowercase								=   FunnelModelTester(self							)
											__lowercase								=   ConfigTester(self	,					config_class=lowerCAmelCase__							)





							def      a__					(					self    :  str							)		->						Tuple:



											"""simple docstring"""


											self.config_tester.run_common_tests()





							def      a__					(					self    :  Union[str, Any]							)		->						Dict:



											"""simple docstring"""


											__lowercase								=   self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*lowerCAmelCase__							)





							def      a__					(					self    :  List[str]							)		->						Optional[Any]:



											"""simple docstring"""


											__lowercase								=   self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_pretraining(*lowerCAmelCase__							)





							def      a__					(					self    :  Optional[Any]							)		->						List[str]:



											"""simple docstring"""


											__lowercase								=   self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase__							)





							def      a__					(					self    :  List[Any]							)		->						int:



											"""simple docstring"""


											__lowercase								=   self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_token_classification(*lowerCAmelCase__							)





							def      a__					(					self    :  Dict							)		->						Any:



											"""simple docstring"""


											__lowercase								=   self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase__							)





							def      a__					(					self    :  str	,					_UpperCAmelCase    :  str							)		->						int:



											"""simple docstring"""


											if hasattr(lowerCAmelCase__	,					'weight'							) and module.weight is not None:
															module.weight.data.fill_(3							)
											if hasattr(lowerCAmelCase__	,					'bias'							) and module.bias is not None:
															module.bias.data.fill_(3							)

											for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
															if hasattr(lowerCAmelCase__	,					lowerCAmelCase__							) and getattr(lowerCAmelCase__	,					lowerCAmelCase__							) is not None:
																			__lowercase								=   getattr(lowerCAmelCase__	,					lowerCAmelCase__							)
																			weight.data.fill_(3							)






@require_torch
class       A__  (		lowerCAmelCase__       ,		unittest.TestCase							):
							lowerCAmelCase__						:		Optional[int]            =				False
							lowerCAmelCase__						:		int            =				False
							lowerCAmelCase__						:		Dict            =				(
							    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
							)





							def      a__					(					self    :  Union[str, Any]							)		->						List[Any]:



											"""simple docstring"""


											__lowercase								=   FunnelModelTester(self	,					base=lowerCAmelCase__							)
											__lowercase								=   ConfigTester(self	,					config_class=lowerCAmelCase__							)





							def      a__					(					self    :  Any							)		->						Tuple:



											"""simple docstring"""


											self.config_tester.run_common_tests()





							def      a__					(					self    :  Optional[int]							)		->						str:



											"""simple docstring"""


											__lowercase								=   self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_base_model(*lowerCAmelCase__							)





							def      a__					(					self    :  List[Any]							)		->						Optional[Any]:



											"""simple docstring"""


											__lowercase								=   self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase__							)





							def      a__					(					self    :  Optional[Any]							)		->						List[Any]:



											"""simple docstring"""


											__lowercase								=   self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_multiple_choice(*lowerCAmelCase__							)





							def      a__					(					self    :  Optional[Any]							)		->						Union[str, Any]:



											"""simple docstring"""


											__lowercase						,   __lowercase								=   self.model_tester.prepare_config_and_inputs_for_common()
											__lowercase								=   True

											for model_class in self.all_model_classes:
															if model_class.__name__ == "FunnelBaseModel":
																			continue
															__lowercase								=   model_class(lowerCAmelCase__							)
															model.to(lowerCAmelCase__							)
															model.train()
															__lowercase								=   self._prepare_for_class(lowerCAmelCase__	,					lowerCAmelCase__	,					return_labels=lowerCAmelCase__							)
															__lowercase								=   model(**lowerCAmelCase__							).loss
															loss.backward()





							def      a__					(					self    :  Optional[Any]	,					_UpperCAmelCase    :  Tuple							)		->						Any:



											"""simple docstring"""


											if hasattr(lowerCAmelCase__	,					'weight'							) and module.weight is not None:
															module.weight.data.fill_(3							)
											if hasattr(lowerCAmelCase__	,					'bias'							) and module.bias is not None:
															module.bias.data.fill_(3							)

											for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
															if hasattr(lowerCAmelCase__	,					lowerCAmelCase__							) and getattr(lowerCAmelCase__	,					lowerCAmelCase__							) is not None:
																			__lowercase								=   getattr(lowerCAmelCase__	,					lowerCAmelCase__							)
																			weight.data.fill_(3							)






@require_torch
@require_sentencepiece
@require_tokenizers
class       A__  (		unittest.TestCase							):





							def      a__					(					self    :  List[Any]							)		->						Any:



											"""simple docstring"""


											__lowercase								=   13
											__lowercase								=   7
											__lowercase								=   torch.arange(0	,					batch_size * sequence_length							).long().reshape(lowerCAmelCase__	,					lowerCAmelCase__							)
											__lowercase								=   [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
											__lowercase								=   torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]							)

											__lowercase								=   FunnelModel.from_pretrained('sgugger/funnel-random-tiny'							)
											__lowercase								=   model(lowerCAmelCase__	,					token_type_ids=lowerCAmelCase__							)[0].abs()

											__lowercase								=   torch.tensor(2_344.8_352							)
											__lowercase								=   torch.tensor(0.8_052							)
											self.assertTrue(torch.allclose(output.sum()	,					lowerCAmelCase__	,					atol=1e-4							)							)
											self.assertTrue(torch.allclose(output.mean()	,					lowerCAmelCase__	,					atol=1e-4							)							)

											__lowercase								=   torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]							)
											__lowercase								=   model(lowerCAmelCase__	,					attention_mask=lowerCAmelCase__	,					token_type_ids=lowerCAmelCase__							)[0].abs()

											__lowercase								=   torch.tensor(2_343.8_425							)
											__lowercase								=   torch.tensor(0.8_049							)
											self.assertTrue(torch.allclose(output.sum()	,					lowerCAmelCase__	,					atol=1e-4							)							)
											self.assertTrue(torch.allclose(output.mean()	,					lowerCAmelCase__	,					atol=1e-4							)							)





							@slow
							def      a__					(					self    :  List[Any]							)		->						str:



											"""simple docstring"""


											__lowercase								=   FunnelTokenizer.from_pretrained('huggingface/funnel-small'							)
											__lowercase								=   FunnelModel.from_pretrained('huggingface/funnel-small'							)
											__lowercase								=   tokenizer('Hello! I am the Funnel Transformer model.'	,					return_tensors='pt'							)
											__lowercase								=   model(**lowerCAmelCase__							)[0]

											__lowercase								=   torch.tensor(235.7_246							)
											__lowercase								=   torch.tensor(0.0_256							)
											self.assertTrue(torch.allclose(output.sum()	,					lowerCAmelCase__	,					atol=1e-4							)							)
											self.assertTrue(torch.allclose(output.mean()	,					lowerCAmelCase__	,					atol=1e-4							)							)






"""simple docstring"""


import random
import sys

__UpperCamelCase										=     '''ABCDEFGHIJKLMNOPQRSTUVWXYZ'''







def    UpperCAmelCase       (						)							-> None:
     snake_case_   			=   input('Enter message: '   )
     snake_case_   			=   'LFWOAYUISVKMNXPBDCRJTQEGHZ'
     snake_case_   			=   input('Encrypt/Decrypt [e/d]: '   )

     check_valid_key(UpperCAmelCase   )

     if resp.lower().startswith('e'   ):
          snake_case_   			=   'encrypt'
          snake_case_   			=   encrypt_message(UpperCAmelCase		,   UpperCAmelCase   )
     elif resp.lower().startswith('d'   ):
          snake_case_   			=   'decrypt'
          snake_case_   			=   decrypt_message(UpperCAmelCase		,   UpperCAmelCase   )

     print(f'\n{mode.title()}ion: \n{translated}'   )







def    UpperCAmelCase       (						UpperCAmelCase   )							-> None:
     snake_case_   			=   list(UpperCAmelCase   )
     snake_case_   			=   list(UpperCAmelCase   )
     key_list.sort()
     letters_list.sort()

     if key_list != letters_list:
          sys.exit('Error in the key or symbol set.'   )







def    UpperCAmelCase       (						UpperCAmelCase		,   UpperCAmelCase   )							-> str:
     return translate_message(UpperCAmelCase		,   UpperCAmelCase		,   'encrypt'   )







def    UpperCAmelCase       (						UpperCAmelCase		,   UpperCAmelCase   )							-> str:
     return translate_message(UpperCAmelCase		,   UpperCAmelCase		,   'decrypt'   )







def    UpperCAmelCase       (						UpperCAmelCase		,   UpperCAmelCase		,   UpperCAmelCase   )							-> str:
     snake_case_   			=   ''
     snake_case_   			=   LETTERS
     snake_case_   			=   key

     if mode == "decrypt":
          snake_case_			,      snake_case_   			=   chars_b, chars_a

     for symbol in message:
          if symbol.upper() in chars_a:
               snake_case_   			=   chars_a.find(symbol.upper()   )
               if symbol.isupper():
                    translated += chars_b[sym_index].upper()
               else:
                    translated += chars_b[sym_index].lower()
          else:
               translated += symbol

     return translated







def    UpperCAmelCase       (						)							-> str:
     snake_case_   			=   list(UpperCAmelCase   )
     random.shuffle(UpperCAmelCase   )
     return "".join(UpperCAmelCase   )


if __name__ == "__main__":
    main()

