






import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import (
						    MODEL_FOR_PRETRAINING_MAPPING,
						    FunnelBaseModel,
						    FunnelForMaskedLM,
						    FunnelForMultipleChoice,
						    FunnelForPreTraining,
						    FunnelForQuestionAnswering,
						    FunnelForSequenceClassification,
						    FunnelForTokenClassification,
						    FunnelModel,
						)

class      UpperCamelCase_							:




						def __init__(    self		,	lowercase		,	lowercase=13		,	lowercase=7		,	lowercase=True		,	lowercase=True		,	lowercase=True		,	lowercase=True		,	lowercase=99		,	lowercase=[1, 1, 2]		,	lowercase=1		,	lowercase=32		,	lowercase=4		,	lowercase=8		,	lowercase=37		,	lowercase="gelu_new"		,	lowercase=0.1		,	lowercase=0.1		,	lowercase=0.0		,	lowercase=512		,	lowercase=3		,	lowercase=0.02		,	lowercase=3		,	lowercase=4		,	lowercase=None		,	lowercase=False		,	)			->						List[str]:
													_a  :    int			=						parent
													_a  :    Tuple			=						batch_size
													_a  :    Any			=						seq_length
													_a  :    Union[str, Any]			=						is_training
													_a  :    Tuple			=						use_input_mask
													_a  :    Any			=						use_token_type_ids
													_a  :    Optional[int]			=						use_labels
													_a  :    Dict			=						vocab_size
													_a  :    List[Any]			=						block_sizes
													_a  :    Dict			=						num_decoder_layers
													_a  :    str			=						d_model
													_a  :    str			=						n_head
													_a  :    str			=						d_head
													_a  :    Union[str, Any]			=						d_inner
													_a  :    Tuple			=						hidden_act
													_a  :    Tuple			=						hidden_dropout
													_a  :    int			=						attention_dropout
													_a  :    int			=						activation_dropout
													_a  :    int			=						max_position_embeddings
													_a  :    Tuple			=						type_vocab_size
													_a  :    Optional[Any]			=						2
													_a  :    int			=						num_labels
													_a  :    int			=						num_choices
													_a  :    Any			=						scope
													_a  :    int			=						initializer_std

													# Used in the tests to check the size of the first attention layer
													_a  :    Dict			=						n_head
													# Used in the tests to check the size of the first hidden state
													_a  :    str			=						self.d_model
													# Used in the tests to check the number of output hidden states/attentions
													_a  :    int			=						sum(self.block_sizes       ) + (0 if base else self.num_decoder_layers)
													# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
													# the last hidden state of the first block (which is the first hidden state of the decoder).
													if not base:
																				_a  :    Dict			=						self.num_hidden_layers + 2




						def      snake_case__(    self       )			->						Dict:
													_a  :    Any			=						ids_tensor([self.batch_size, self.seq_length]		,	self.vocab_size       )

													_a  :    Optional[Any]			=						None
													if self.use_input_mask:
																				_a  :    int			=						ids_tensor([self.batch_size, self.seq_length]		,	vocab_size=2       )

													_a  :    Dict			=						None
													if self.use_token_type_ids:
																				_a  :    int			=						ids_tensor([self.batch_size, self.seq_length]		,	self.type_vocab_size       )

													_a  :    List[Any]			=						None
													_a  :    List[str]			=						None
													_a  :    Dict			=						None
													if self.use_labels:
																				_a  :    Optional[Any]			=						ids_tensor([self.batch_size]		,	self.type_sequence_label_size       )
																				_a  :    str			=						ids_tensor([self.batch_size, self.seq_length]		,	self.num_labels       )
																				_a  :    str			=						ids_tensor([self.batch_size]		,	self.num_choices       )
																				_a  :    Any			=						ids_tensor([self.batch_size, self.seq_length]		,	1       )

													_a  :    Optional[int]			=						self.get_config()

													return (
													    config,
													    input_ids,
													    token_type_ids,
													    input_mask,
													    sequence_labels,
													    token_labels,
													    choice_labels,
													    fake_token_labels,
													)




						def      snake_case__(    self       )			->						str:
													return FunnelConfig(
													    vocab_size=self.vocab_size		,	block_sizes=self.block_sizes		,	num_decoder_layers=self.num_decoder_layers		,	d_model=self.d_model		,	n_head=self.n_head		,	d_head=self.d_head		,	d_inner=self.d_inner		,	hidden_act=self.hidden_act		,	hidden_dropout=self.hidden_dropout		,	attention_dropout=self.attention_dropout		,	activation_dropout=self.activation_dropout		,	max_position_embeddings=self.max_position_embeddings		,	type_vocab_size=self.type_vocab_size		,	initializer_std=self.initializer_std		,	)




						def      snake_case__(    self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)			->						int:
													_a  :    Optional[Any]			=						FunnelModel(config=_lowerCamelCase       )
													model.to(_lowerCamelCase       )
													model.eval()
													_a  :    Tuple			=						model(_lowerCamelCase		,	attention_mask=_lowerCamelCase		,	token_type_ids=_lowerCamelCase       )
													_a  :    Union[str, Any]			=						model(_lowerCamelCase		,	token_type_ids=_lowerCamelCase       )
													_a  :    Any			=						model(_lowerCamelCase       )
													self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, self.seq_length, self.d_model)       )

													_a  :    Optional[int]			=						False
													_a  :    str			=						model(_lowerCamelCase       )
													self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, self.seq_length, self.d_model)       )

													_a  :    Optional[int]			=						False
													_a  :    Dict			=						model(_lowerCamelCase       )
													self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, self.seq_length, self.d_model)       )




						def      snake_case__(    self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)			->						Tuple:
													_a  :    Dict			=						FunnelBaseModel(config=_lowerCamelCase       )
													model.to(_lowerCamelCase       )
													model.eval()
													_a  :    List[str]			=						model(_lowerCamelCase		,	attention_mask=_lowerCamelCase		,	token_type_ids=_lowerCamelCase       )
													_a  :    str			=						model(_lowerCamelCase		,	token_type_ids=_lowerCamelCase       )
													_a  :    int			=						model(_lowerCamelCase       )
													self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, 2, self.d_model)       )

													_a  :    List[Any]			=						False
													_a  :    Union[str, Any]			=						model(_lowerCamelCase       )
													self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, 3, self.d_model)       )

													_a  :    str			=						False
													_a  :    Optional[Any]			=						model(_lowerCamelCase       )
													self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, 2, self.d_model)       )




						def      snake_case__(    self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)			->						List[str]:
													_a  :    List[str]			=						self.num_labels
													_a  :    Optional[int]			=						FunnelForPreTraining(config=_lowerCamelCase       )
													model.to(_lowerCamelCase       )
													model.eval()
													_a  :    List[str]			=						model(_lowerCamelCase		,	attention_mask=_lowerCamelCase		,	token_type_ids=_lowerCamelCase		,	labels=_lowerCamelCase       )
													self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.seq_length)       )




						def      snake_case__(    self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)			->						Any:
													_a  :    List[Any]			=						FunnelForMaskedLM(config=_lowerCamelCase       )
													model.to(_lowerCamelCase       )
													model.eval()
													_a  :    List[str]			=						model(_lowerCamelCase		,	attention_mask=_lowerCamelCase		,	token_type_ids=_lowerCamelCase		,	labels=_lowerCamelCase       )
													self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.seq_length, self.vocab_size)       )




						def      snake_case__(    self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)			->						List[str]:
													_a  :    str			=						self.num_labels
													_a  :    str			=						FunnelForSequenceClassification(_lowerCamelCase       )
													model.to(_lowerCamelCase       )
													model.eval()
													_a  :    Union[str, Any]			=						model(_lowerCamelCase		,	attention_mask=_lowerCamelCase		,	token_type_ids=_lowerCamelCase		,	labels=_lowerCamelCase       )
													self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.num_labels)       )




						def      snake_case__(    self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)			->						Dict:
													_a  :    List[str]			=						self.num_choices
													_a  :    Tuple			=						FunnelForMultipleChoice(config=_lowerCamelCase       )
													model.to(_lowerCamelCase       )
													model.eval()
													_a  :    Optional[Any]			=						input_ids.unsqueeze(1       ).expand(-1		,	self.num_choices		,	-1       ).contiguous()
													_a  :    Optional[Any]			=						token_type_ids.unsqueeze(1       ).expand(-1		,	self.num_choices		,	-1       ).contiguous()
													_a  :    Any			=						input_mask.unsqueeze(1       ).expand(-1		,	self.num_choices		,	-1       ).contiguous()
													_a  :    Union[str, Any]			=						model(
													    _lowerCamelCase		,	attention_mask=_lowerCamelCase		,	token_type_ids=_lowerCamelCase		,	labels=_lowerCamelCase		,	)
													self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.num_choices)       )




						def      snake_case__(    self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)			->						Optional[Any]:
													_a  :    List[Any]			=						self.num_labels
													_a  :    Union[str, Any]			=						FunnelForTokenClassification(config=_lowerCamelCase       )
													model.to(_lowerCamelCase       )
													model.eval()
													_a  :    List[str]			=						model(_lowerCamelCase		,	attention_mask=_lowerCamelCase		,	token_type_ids=_lowerCamelCase		,	labels=_lowerCamelCase       )
													self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.seq_length, self.num_labels)       )




						def      snake_case__(    self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)			->						Optional[int]:
													_a  :    Tuple			=						FunnelForQuestionAnswering(config=_lowerCamelCase       )
													model.to(_lowerCamelCase       )
													model.eval()
													_a  :    List[str]			=						model(
													    _lowerCamelCase		,	attention_mask=_lowerCamelCase		,	token_type_ids=_lowerCamelCase		,	start_positions=_lowerCamelCase		,	end_positions=_lowerCamelCase		,	)
													self.parent.assertEqual(result.start_logits.shape		,	(self.batch_size, self.seq_length)       )
													self.parent.assertEqual(result.end_logits.shape		,	(self.batch_size, self.seq_length)       )







						def      snake_case__(    self       )			->						Dict:
													_a  :    Union[str, Any]			=						self.prepare_config_and_inputs()
													(
													    _a
													)  :    Any			=						config_and_inputs
													_a  :    List[str]			=						{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
													return config, inputs_dict

@require_torch
class      UpperCamelCase_							(				UpperCamelCase ,       UpperCamelCase ,       unittest.TestCase       ):
						lowercase									=  False
						lowercase									=  False
						lowercase									=  (
						    (
						        FunnelModel,
						        FunnelForMaskedLM,
						        FunnelForPreTraining,
						        FunnelForQuestionAnswering,
						        FunnelForTokenClassification,
						    )
						    if is_torch_available()
						    else ()
						)
						lowercase									=  (
						    {
						        """feature-extraction""": (FunnelBaseModel, FunnelModel),
						        """fill-mask""": FunnelForMaskedLM,
						        """question-answering""": FunnelForQuestionAnswering,
						        """text-classification""": FunnelForSequenceClassification,
						        """token-classification""": FunnelForTokenClassification,
						        """zero-shot""": FunnelForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)




						def      snake_case__(    self		,	lowercase		,	lowercase		,	lowercase=False       )			->						Any:
													_a  :    List[Any]			=						super()._prepare_for_class(_lowerCamelCase		,	_lowerCamelCase		,	return_labels=_lowerCamelCase       )

													if return_labels:
																				if model_class in get_values(_lowerCamelCase       ):
																											_a  :    Any			=						torch.zeros(
																											    (self.model_tester.batch_size, self.model_tester.seq_length)		,	dtype=torch.long		,	device=_lowerCamelCase       )
													return inputs_dict




						def      snake_case__(    self       )			->						Optional[int]:
													_a  :    Dict			=						FunnelModelTester(self       )
													_a  :    List[str]			=						ConfigTester(self		,	config_class=_lowerCamelCase       )




						def      snake_case__(    self       )			->						int:
													self.config_tester.run_common_tests()




						def      snake_case__(    self       )			->						List[Any]:
													_a  :    Dict			=						self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_model(*_lowerCamelCase       )




						def      snake_case__(    self       )			->						List[Any]:
													_a  :    Tuple			=						self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_pretraining(*_lowerCamelCase       )




						def      snake_case__(    self       )			->						List[Any]:
													_a  :    Tuple			=						self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_masked_lm(*_lowerCamelCase       )




						def      snake_case__(    self       )			->						List[str]:
													_a  :    int			=						self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_token_classification(*_lowerCamelCase       )




						def      snake_case__(    self       )			->						Optional[int]:
													_a  :    Dict			=						self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_question_answering(*_lowerCamelCase       )







						def      snake_case__(    self		,	lowercase       )			->						List[Any]:
													if hasattr(_lowerCamelCase		,	'''weight'''       ) and module.weight is not None:
																				module.weight.data.fill_(3       )
													if hasattr(_lowerCamelCase		,	'''bias'''       ) and module.bias is not None:
																				module.bias.data.fill_(3       )

													for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
																				if hasattr(_lowerCamelCase		,	_lowerCamelCase       ) and getattr(_lowerCamelCase		,	_lowerCamelCase       ) is not None:
																											_a  :    Union[str, Any]			=						getattr(_lowerCamelCase		,	_lowerCamelCase       )
																											weight.data.fill_(3       )

@require_torch
class      UpperCamelCase_							(				UpperCamelCase ,       unittest.TestCase       ):
						lowercase									=  False
						lowercase									=  False
						lowercase									=  (
						    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
						)




						def      snake_case__(    self       )			->						Tuple:
													_a  :    Optional[int]			=						FunnelModelTester(self		,	base=_lowerCamelCase       )
													_a  :    int			=						ConfigTester(self		,	config_class=_lowerCamelCase       )




						def      snake_case__(    self       )			->						Optional[int]:
													self.config_tester.run_common_tests()




						def      snake_case__(    self       )			->						int:
													_a  :    Optional[Any]			=						self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_base_model(*_lowerCamelCase       )




						def      snake_case__(    self       )			->						Tuple:
													_a  :    Any			=						self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_sequence_classification(*_lowerCamelCase       )




						def      snake_case__(    self       )			->						Union[str, Any]:
													_a  :    Tuple			=						self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_multiple_choice(*_lowerCamelCase       )




						def      snake_case__(    self       )			->						str:
													_a  :    Any			=						self.model_tester.prepare_config_and_inputs_for_common()
													_a  :    Optional[int]			=						True

													for model_class in self.all_model_classes:
																				if model_class.__name__ == "FunnelBaseModel":
																											continue
																				_a  :    List[str]			=						model_class(_lowerCamelCase       )
																				model.to(_lowerCamelCase       )
																				model.train()
																				_a  :    Union[str, Any]			=						self._prepare_for_class(_lowerCamelCase		,	_lowerCamelCase		,	return_labels=_lowerCamelCase       )
																				_a  :    Any			=						model(**_lowerCamelCase       ).loss
																				loss.backward()







						def      snake_case__(    self		,	lowercase       )			->						str:
													if hasattr(_lowerCamelCase		,	'''weight'''       ) and module.weight is not None:
																				module.weight.data.fill_(3       )
													if hasattr(_lowerCamelCase		,	'''bias'''       ) and module.bias is not None:
																				module.bias.data.fill_(3       )

													for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
																				if hasattr(_lowerCamelCase		,	_lowerCamelCase       ) and getattr(_lowerCamelCase		,	_lowerCamelCase       ) is not None:
																											_a  :    List[Any]			=						getattr(_lowerCamelCase		,	_lowerCamelCase       )
																											weight.data.fill_(3       )

@require_torch
@require_sentencepiece
@require_tokenizers
class      UpperCamelCase_							(				unittest.TestCase       ):




						def      snake_case__(    self       )			->						Dict:
													_a  :    int			=						13
													_a  :    Union[str, Any]			=						7
													_a  :    List[Any]			=						torch.arange(0		,	batch_size * sequence_length       ).long().reshape(_lowerCamelCase		,	_lowerCamelCase       )
													_a  :    Any			=						[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
													_a  :    Union[str, Any]			=						torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]       )

													_a  :    Union[str, Any]			=						FunnelModel.from_pretrained('''sgugger/funnel-random-tiny'''       )
													_a  :    Dict			=						model(_lowerCamelCase		,	token_type_ids=_lowerCamelCase       )[0].abs()

													_a  :    str			=						torch.tensor(2_344.8_352       )
													_a  :    Any			=						torch.tensor(0.8052       )
													self.assertTrue(torch.allclose(output.sum()		,	_lowerCamelCase		,	atol=1e-4       )       )
													self.assertTrue(torch.allclose(output.mean()		,	_lowerCamelCase		,	atol=1e-4       )       )

													_a  :    str			=						torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]       )
													_a  :    Tuple			=						model(_lowerCamelCase		,	attention_mask=_lowerCamelCase		,	token_type_ids=_lowerCamelCase       )[0].abs()

													_a  :    Optional[int]			=						torch.tensor(2_343.8_425       )
													_a  :    Dict			=						torch.tensor(0.8049       )
													self.assertTrue(torch.allclose(output.sum()		,	_lowerCamelCase		,	atol=1e-4       )       )
													self.assertTrue(torch.allclose(output.mean()		,	_lowerCamelCase		,	atol=1e-4       )       )







						@slow
						def      snake_case__(    self       )			->						List[str]:
													_a  :    Dict			=						FunnelTokenizer.from_pretrained('''huggingface/funnel-small'''       )
													_a  :    int			=						FunnelModel.from_pretrained('''huggingface/funnel-small'''       )
													_a  :    List[Any]			=						tokenizer('''Hello! I am the Funnel Transformer model.'''		,	return_tensors='''pt'''       )
													_a  :    int			=						model(**_lowerCamelCase       )[0]

													_a  :    Dict			=						torch.tensor(235.7_246       )
													_a  :    str			=						torch.tensor(0.0256       )
													self.assertTrue(torch.allclose(output.sum()		,	_lowerCamelCase		,	atol=1e-4       )       )
													self.assertTrue(torch.allclose(output.mean()		,	_lowerCamelCase		,	atol=1e-4       )       )

import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device


if is_torch_available():
				import torch

				from transformers import CamembertModel
@require_torch
@require_sentencepiece
@require_tokenizers
class    _UpperCamelCase(   unittest.TestCase				):

						@slow
						def 						a__ (					self		:			Optional[Any]			):
													_UpperCAmelCase  :			Tuple	       =		CamembertModel.from_pretrained("camembert-base"			)
													model.to(_lowerCamelCase			)

													_UpperCAmelCase  :			List[Any]	       =		torch.tensor(
													    [[5, 1_21, 11, 6_60, 16, 7_30, 2_55_43, 1_10, 83, 6]]	,						device=_lowerCamelCase	,						dtype=torch.long	,						)  # J'aime le camembert !
													with torch.no_grad():
																				_UpperCAmelCase  :			Dict	       =		model(_lowerCamelCase			)["last_hidden_state"]
													_UpperCAmelCase  :			Optional[int]	       =		torch.Size((1, 10, 7_68)			)
													self.assertEqual(output.shape	,						_lowerCamelCase			)
													# compare the actual values for a slice.
													_UpperCAmelCase  :			List[str]	       =		torch.tensor(
													    [[[-0.02_54, 0.02_35, 0.10_27], [0.06_06, -0.18_11, -0.04_18], [-0.15_61, -0.11_27, 0.26_87]]]	,						device=_lowerCamelCase	,						dtype=torch.float	,						)
													# camembert = torch.hub.load('pytorch/fairseq', 'camembert.v0')
													# camembert.eval()
													# expected_slice = roberta.model.forward(input_ids)[0][:, :3, :3].detach()

													self.assertTrue(torch.allclose(output[:, :3, :3]	,						_lowerCamelCase	,						atol=1E-4			)			)




