


"""simple docstring"""


import argparse

import torch
from saprl.hub import distilhubert

from transformers import HubertConfig, HubertModel, WavaVecaFeatureExtractor, logging


logging.set_verbosity_info()
a_											=     logging.get_logger(__name__)

a_											=     {
    "post_extract_proj": "feature_projection.projection",
    "encoder.pos_conv.0": "encoder.pos_conv_embed.conv",
    "self_attn.k_proj": "encoder.layers.*.attention.k_proj",
    "self_attn.v_proj": "encoder.layers.*.attention.v_proj",
    "self_attn.q_proj": "encoder.layers.*.attention.q_proj",
    "self_attn.out_proj": "encoder.layers.*.attention.out_proj",
    "self_attn_layer_norm": "encoder.layers.*.layer_norm",
    "fc1": "encoder.layers.*.feed_forward.intermediate_dense",
    "fc2": "encoder.layers.*.feed_forward.output_dense",
    "final_layer_norm": "encoder.layers.*.final_layer_norm",
    "encoder.layer_norm": "encoder.layer_norm",
    "mask_emb": "masked_spec_embed",
}






def   a__			(					__lowercase  ,      __lowercase  ,      __lowercase  ,      __lowercase  ,      __lowercase )						->						List[str]:
				for attribute in key.split("." ):
								_A       	=      getattr(__lowercase  ,      __lowercase )

				if weight_type is not None:
								_A       	=      getattr(__lowercase  ,      __lowercase ).shape
				else:
								_A       	=      hf_pointer.shape

				assert hf_shape == value.shape, (
				    f"""Shape of hf {key + '.' + weight_type if weight_type is not None else ''} is {hf_shape}, but should be"""
				    f""" {value.shape} for {full_name}"""
				)

				if weight_type == "weight":
								_A       	=      value
				elif weight_type == "weight_g":
								_A       	=      value
				elif weight_type == "weight_v":
								_A       	=      value
				elif weight_type == "bias":
								_A       	=      value
				else:
								_A       	=      value

				logger.info(f"""{key + '.' + weight_type if weight_type is not None else ''} was initialized from {full_name}.""" )






def   a__			(					__lowercase  ,      __lowercase )						->						Optional[int]:
				_A       	=      []
				_A       	=      fairseq_model.state_dict()

				_A       	=      hf_model.feature_extractor

				for name, value in fairseq_dict.items():
								_A       	=      False
								if "conv_layers" in name:
												load_conv_layer(
												    __lowercase  ,      __lowercase  ,      __lowercase  ,      __lowercase  ,      hf_model.config.feat_extract_norm == "group"  ,      )
												_A       	=      True
								else:
												for key, mapped_key in MAPPING.items():
																_A       	=      mapped_key

																if key in name:
																				_A       	=      True
																				if "*" in mapped_key:
																								_A       	=      name.split(__lowercase )[0].split("." )[-2]
																								_A       	=      mapped_key.replace("*"  ,      __lowercase )
																				if "weight_g" in name:
																								_A       	=      '''weight_g'''
																				elif "weight_v" in name:
																								_A       	=      '''weight_v'''
																				elif "weight" in name:
																								_A       	=      '''weight'''
																				elif "bias" in name:
																								_A       	=      '''bias'''
																				else:
																								_A       	=      None
																				set_recursively(__lowercase  ,      __lowercase  ,      __lowercase  ,      __lowercase  ,      __lowercase )
																continue
								if not is_used:
												unused_weights.append(__lowercase )

				logger.warning(f"""Unused weights: {unused_weights}""" )






def   a__			(					__lowercase  ,      __lowercase  ,      __lowercase  ,      __lowercase  ,      __lowercase )						->						Dict:
				_A       	=      full_name.split("conv_layers." )[-1]
				_A       	=      name.split("." )
				_A       	=      int(items[0] )
				_A       	=      int(items[1] )

				if type_id == 0:
								if "bias" in name:
												assert value.shape == feature_extractor.conv_layers[layer_id].conv.bias.data.shape, (
												    f"""{full_name} has size {value.shape}, but"""
												    f""" {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found."""
												)
												_A       	=      value
												logger.info(f"""Feat extract conv layer {layer_id} was initialized from {full_name}.""" )
								elif "weight" in name:
												assert value.shape == feature_extractor.conv_layers[layer_id].conv.weight.data.shape, (
												    f"""{full_name} has size {value.shape}, but"""
												    f""" {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found."""
												)
												_A       	=      value
												logger.info(f"""Feat extract conv layer {layer_id} was initialized from {full_name}.""" )
				elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
								if "bias" in name:
												assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape, (
												    f"""{full_name} has size {value.shape}, but {feature_extractor[layer_id].layer_norm.bias.data.shape} was"""
												    " found."
												)
												_A       	=      value
												logger.info(f"""Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.""" )
								elif "weight" in name:
												assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape, (
												    f"""{full_name} has size {value.shape}, but"""
												    f""" {feature_extractor[layer_id].layer_norm.weight.data.shape} was found."""
												)
												_A       	=      value
												logger.info(f"""Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.""" )
				else:
								unused_weights.append(__lowercase )






def   a__			(					__lowercase )						->						Optional[Any]:
				_A       	=      HubertConfig()
				_A       	=      model.config

				_A       	=      fs_config.activation_dropout
				_A       	=      False
				_A       	=      fs_config.attention_dropout
				_A       	=      False
				_A       	=      eval(fs_config.extractor_conv_feature_layers )
				_A       	=      [x[0] for x in conv_layers]
				_A       	=      [x[1] for x in conv_layers]
				_A       	=      [x[2] for x in conv_layers]
				_A       	=      '''gelu'''
				_A       	=      '''layer''' if fs_config.extractor_mode == '''layer_norm''' else '''group'''
				_A       	=      False
				_A       	=      0.0
				_A       	=      0.0
				_A       	=      fs_config.activation_fn
				_A       	=      fs_config.dropout
				_A       	=      fs_config.encoder_embed_dim
				_A       	=      0.02
				_A       	=      fs_config.encoder_ffn_embed_dim
				_A       	=      1E-5
				_A       	=      0.0
				_A       	=      fs_config.encoder_attention_heads
				_A       	=      fs_config.conv_pos_groups
				_A       	=      fs_config.conv_pos
				_A       	=      len(__lowercase )
				_A       	=      fs_config.encoder_layers

				return config






@torch.no_grad()
def   a__			(					__lowercase  ,      __lowercase=None )						->						Optional[int]:
				_A       	=      distilhubert().model.model

				if config_path is not None:
								_A       	=      HubertConfig.from_pretrained(__lowercase )
				else:
								_A       	=      convert_config(__lowercase )
				_A       	=      model.eval()

				_A       	=      WavaVecaFeatureExtractor(
				    feature_size=1  ,      sampling_rate=1_6000  ,      padding_value=0  ,      do_normalize=__lowercase  ,      return_attention_mask=__lowercase  ,      )
				_A       	=      HubertModel(__lowercase )

				recursively_load_weights(__lowercase  ,      __lowercase )

				feature_extractor.save_pretrained(__lowercase )
				hf_model.save_pretrained(__lowercase )


if __name__ == "__main__":
		a_											=     argparse.ArgumentParser()
		parser.add_argument("--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model.")
		parser.add_argument("--config_path", default=None, type=str, help="Path to hf config.json of model to convert")
		a_											=     parser.parse_args()
		convert_hubert_checkpoint(args.pytorch_dump_folder_path, args.config_path)


import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         MODEL_FOR_PRETRAINING_MAPPING,
         FunnelBaseModel,
         FunnelForMaskedLM,
         FunnelForMultipleChoice,
         FunnelForPreTraining,
         FunnelForQuestionAnswering,
         FunnelForSequenceClassification,
         FunnelForTokenClassification,
         FunnelModel,
     )





class    __UpperCAmelCase	:



   def __init__(				self				:	List[Any],						__A				:	str,						__A				:	Union[str, Any]=1_3,						__A				:	Union[str, Any]=7,						__A				:	List[str]=True,						__A				:	Optional[Any]=True,						__A				:	Tuple=True,						__A				:	Any=True,						__A				:	Dict=9_9,						__A				:	Union[str, Any]=[1, 1, 2],						__A				:	Tuple=1,						__A				:	int=3_2,						__A				:	List[str]=4,						__A				:	List[str]=8,						__A				:	int=3_7,						__A				:	Optional[int]="gelu_new",						__A				:	List[Any]=0.1,						__A				:	str=0.1,						__A				:	int=0.0,						__A				:	Union[str, Any]=5_1_2,						__A				:	int=3,						__A				:	str=0.0_2,						__A				:	Tuple=3,						__A				:	Any=4,						__A				:	Any=None,						__A				:	Any=False,						):
     UpperCAmelCase			:				str										=					parent
     UpperCAmelCase			:				Optional[Any]										=					batch_size
     UpperCAmelCase			:				str										=					seq_length
     UpperCAmelCase			:				str										=					is_training
     UpperCAmelCase			:				Tuple										=					use_input_mask
     UpperCAmelCase			:				Union[str, Any]										=					use_token_type_ids
     UpperCAmelCase			:				Dict										=					use_labels
     UpperCAmelCase			:				Union[str, Any]										=					vocab_size
     UpperCAmelCase			:				Tuple										=					block_sizes
     UpperCAmelCase			:				List[Any]										=					num_decoder_layers
     UpperCAmelCase			:				Union[str, Any]										=					d_model
     UpperCAmelCase			:				Tuple										=					n_head
     UpperCAmelCase			:				Dict										=					d_head
     UpperCAmelCase			:				Optional[int]										=					d_inner
     UpperCAmelCase			:				Optional[int]										=					hidden_act
     UpperCAmelCase			:				int										=					hidden_dropout
     UpperCAmelCase			:				Any										=					attention_dropout
     UpperCAmelCase			:				str										=					activation_dropout
     UpperCAmelCase			:				Dict										=					max_position_embeddings
     UpperCAmelCase			:				Optional[Any]										=					type_vocab_size
     UpperCAmelCase			:				Any										=					2
     UpperCAmelCase			:				List[Any]										=					num_labels
     UpperCAmelCase			:				str										=					num_choices
     UpperCAmelCase			:				str										=					scope
     UpperCAmelCase			:				int										=					initializer_std

     # Used in the tests to check the size of the first attention layer
     UpperCAmelCase			:				int										=					n_head
     # Used in the tests to check the size of the first hidden state
     UpperCAmelCase			:				Dict										=					self.d_model
     # Used in the tests to check the number of output hidden states/attentions
     UpperCAmelCase			:				List[str]										=					sum(self.block_sizes		) + (0 if base else self.num_decoder_layers)
     # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
     # the last hidden state of the first block (which is the first hidden state of the decoder).
     if not base:
       UpperCAmelCase			:				List[str]										=					self.num_hidden_layers + 2



   def 			__magic_name__       (				self				:	List[str]		):
     UpperCAmelCase			:				Dict										=					ids_tensor([self.batch_size, self.seq_length],						self.vocab_size		)

     UpperCAmelCase			:				Optional[int]										=					None
     if self.use_input_mask:
       UpperCAmelCase			:				List[str]										=					ids_tensor([self.batch_size, self.seq_length],						vocab_size=2		)

     UpperCAmelCase			:				int										=					None
     if self.use_token_type_ids:
       UpperCAmelCase			:				Dict										=					ids_tensor([self.batch_size, self.seq_length],						self.type_vocab_size		)

     UpperCAmelCase			:				List[Any]										=					None
     UpperCAmelCase			:				Optional[int]										=					None
     UpperCAmelCase			:				str										=					None
     if self.use_labels:
       UpperCAmelCase			:				List[Any]										=					ids_tensor([self.batch_size],						self.type_sequence_label_size		)
       UpperCAmelCase			:				Optional[int]										=					ids_tensor([self.batch_size, self.seq_length],						self.num_labels		)
       UpperCAmelCase			:				List[str]										=					ids_tensor([self.batch_size],						self.num_choices		)
       UpperCAmelCase			:				Tuple										=					ids_tensor([self.batch_size, self.seq_length],						1		)

     UpperCAmelCase			:				Dict										=					self.get_config()

     return (
         config,
         input_ids,
         token_type_ids,
         input_mask,
         sequence_labels,
         token_labels,
         choice_labels,
         fake_token_labels,
     )



   def 			__magic_name__       (				self				:	List[Any]		):
     return FunnelConfig(
         vocab_size=self.vocab_size,						block_sizes=self.block_sizes,						num_decoder_layers=self.num_decoder_layers,						d_model=self.d_model,						n_head=self.n_head,						d_head=self.d_head,						d_inner=self.d_inner,						hidden_act=self.hidden_act,						hidden_dropout=self.hidden_dropout,						attention_dropout=self.attention_dropout,						activation_dropout=self.activation_dropout,						max_position_embeddings=self.max_position_embeddings,						type_vocab_size=self.type_vocab_size,						initializer_std=self.initializer_std,						)



   def 			__magic_name__       (				self				:	Union[str, Any],						__A				:	List[str],						__A				:	Dict,						__A				:	Optional[int],						__A				:	List[Any],						__A				:	List[Any],						__A				:	Optional[int],						__A				:	Dict,						__A				:	Dict,						):
     UpperCAmelCase			:				List[str]										=					FunnelModel(config=__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				List[Any]										=					model(__A,						attention_mask=__A,						token_type_ids=__A		)
     UpperCAmelCase			:				Optional[int]										=					model(__A,						token_type_ids=__A		)
     UpperCAmelCase			:				Optional[int]										=					model(__A		)
     self.parent.assertEqual(result.last_hidden_state.shape,						(self.batch_size, self.seq_length, self.d_model)		)

     UpperCAmelCase			:				Optional[int]										=					False
     UpperCAmelCase			:				List[str]										=					model(__A		)
     self.parent.assertEqual(result.last_hidden_state.shape,						(self.batch_size, self.seq_length, self.d_model)		)

     UpperCAmelCase			:				List[str]										=					False
     UpperCAmelCase			:				Dict										=					model(__A		)
     self.parent.assertEqual(result.last_hidden_state.shape,						(self.batch_size, self.seq_length, self.d_model)		)



   def 			__magic_name__       (				self				:	Dict,						__A				:	str,						__A				:	List[Any],						__A				:	Union[str, Any],						__A				:	Union[str, Any],						__A				:	Tuple,						__A				:	Tuple,						__A				:	List[Any],						__A				:	Dict,						):
     UpperCAmelCase			:				List[Any]										=					FunnelBaseModel(config=__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				int										=					model(__A,						attention_mask=__A,						token_type_ids=__A		)
     UpperCAmelCase			:				List[Any]										=					model(__A,						token_type_ids=__A		)
     UpperCAmelCase			:				Optional[Any]										=					model(__A		)
     self.parent.assertEqual(result.last_hidden_state.shape,						(self.batch_size, 2, self.d_model)		)

     UpperCAmelCase			:				str										=					False
     UpperCAmelCase			:				Optional[Any]										=					model(__A		)
     self.parent.assertEqual(result.last_hidden_state.shape,						(self.batch_size, 3, self.d_model)		)

     UpperCAmelCase			:				Tuple										=					False
     UpperCAmelCase			:				Optional[Any]										=					model(__A		)
     self.parent.assertEqual(result.last_hidden_state.shape,						(self.batch_size, 2, self.d_model)		)



   def 			__magic_name__       (				self				:	Optional[int],						__A				:	Tuple,						__A				:	Dict,						__A				:	str,						__A				:	Dict,						__A				:	Union[str, Any],						__A				:	List[Any],						__A				:	Any,						__A				:	str,						):
     UpperCAmelCase			:				int										=					self.num_labels
     UpperCAmelCase			:				int										=					FunnelForPreTraining(config=__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				List[Any]										=					model(__A,						attention_mask=__A,						token_type_ids=__A,						labels=__A		)
     self.parent.assertEqual(result.logits.shape,						(self.batch_size, self.seq_length)		)



   def 			__magic_name__       (				self				:	List[str],						__A				:	Optional[int],						__A				:	Union[str, Any],						__A				:	int,						__A				:	Union[str, Any],						__A				:	Optional[int],						__A				:	str,						__A				:	int,						__A				:	str,						):
     UpperCAmelCase			:				List[Any]										=					FunnelForMaskedLM(config=__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				Optional[int]										=					model(__A,						attention_mask=__A,						token_type_ids=__A,						labels=__A		)
     self.parent.assertEqual(result.logits.shape,						(self.batch_size, self.seq_length, self.vocab_size)		)



   def 			__magic_name__       (				self				:	Optional[int],						__A				:	Optional[Any],						__A				:	Optional[Any],						__A				:	Dict,						__A				:	Any,						__A				:	Optional[Any],						__A				:	Tuple,						__A				:	Union[str, Any],						__A				:	List[str],						):
     UpperCAmelCase			:				str										=					self.num_labels
     UpperCAmelCase			:				Optional[int]										=					FunnelForSequenceClassification(__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				Dict										=					model(__A,						attention_mask=__A,						token_type_ids=__A,						labels=__A		)
     self.parent.assertEqual(result.logits.shape,						(self.batch_size, self.num_labels)		)



   def 			__magic_name__       (				self				:	Union[str, Any],						__A				:	Any,						__A				:	List[str],						__A				:	str,						__A				:	List[Any],						__A				:	List[str],						__A				:	Optional[Any],						__A				:	List[Any],						__A				:	List[str],						):
     UpperCAmelCase			:				str										=					self.num_choices
     UpperCAmelCase			:				Optional[int]										=					FunnelForMultipleChoice(config=__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				List[Any]										=					input_ids.unsqueeze(1		).expand(-1,						self.num_choices,						-1		).contiguous()
     UpperCAmelCase			:				List[Any]										=					token_type_ids.unsqueeze(1		).expand(-1,						self.num_choices,						-1		).contiguous()
     UpperCAmelCase			:				List[Any]										=					input_mask.unsqueeze(1		).expand(-1,						self.num_choices,						-1		).contiguous()
     UpperCAmelCase			:				List[str]										=					model(
         __A,						attention_mask=__A,						token_type_ids=__A,						labels=__A,						)
     self.parent.assertEqual(result.logits.shape,						(self.batch_size, self.num_choices)		)



   def 			__magic_name__       (				self				:	Dict,						__A				:	Tuple,						__A				:	int,						__A				:	Any,						__A				:	Optional[Any],						__A				:	Optional[Any],						__A				:	Tuple,						__A				:	str,						__A				:	List[str],						):
     UpperCAmelCase			:				Union[str, Any]										=					self.num_labels
     UpperCAmelCase			:				Optional[int]										=					FunnelForTokenClassification(config=__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				Any										=					model(__A,						attention_mask=__A,						token_type_ids=__A,						labels=__A		)
     self.parent.assertEqual(result.logits.shape,						(self.batch_size, self.seq_length, self.num_labels)		)



   def 			__magic_name__       (				self				:	int,						__A				:	Optional[int],						__A				:	List[str],						__A				:	List[Any],						__A				:	int,						__A				:	str,						__A				:	Any,						__A				:	Union[str, Any],						__A				:	str,						):
     UpperCAmelCase			:				str										=					FunnelForQuestionAnswering(config=__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				Optional[Any]										=					model(
         __A,						attention_mask=__A,						token_type_ids=__A,						start_positions=__A,						end_positions=__A,						)
     self.parent.assertEqual(result.start_logits.shape,						(self.batch_size, self.seq_length)		)
     self.parent.assertEqual(result.end_logits.shape,						(self.batch_size, self.seq_length)		)





   def 			__magic_name__       (				self				:	Optional[Any]		):
     UpperCAmelCase			:				Dict										=					self.prepare_config_and_inputs()
     (
         (
         UpperCAmelCase
     )					,		(
         UpperCAmelCase
     )					,		(
         UpperCAmelCase
     )					,		(
         UpperCAmelCase
     )					,		(
         UpperCAmelCase
     )					,		(
         UpperCAmelCase
     )					,		(
         UpperCAmelCase
     )					,		(
         UpperCAmelCase
     )					,		
     )			:				Optional[int]										=					config_and_inputs
     UpperCAmelCase			:				int										=					{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
     return config, inputs_dict






@require_torch
class    __UpperCAmelCase	(			lowerCamelCase__     ,    lowerCamelCase__     ,    unittest.TestCase		):
   UpperCamelCase  	=	False
   UpperCamelCase  	=	False
   UpperCamelCase  	=	(
       (
           FunnelModel,
           FunnelForMaskedLM,
           FunnelForPreTraining,
           FunnelForQuestionAnswering,
           FunnelForTokenClassification,
       )
       if is_torch_available()
       else ()
   )
   UpperCamelCase  	=	(
       {
           """feature-extraction""": (FunnelBaseModel, FunnelModel),
           """fill-mask""": FunnelForMaskedLM,
           """question-answering""": FunnelForQuestionAnswering,
           """text-classification""": FunnelForSequenceClassification,
           """token-classification""": FunnelForTokenClassification,
           """zero-shot""": FunnelForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )



   def 			__magic_name__       (				self				:	Dict,						__A				:	List[Any],						__A				:	str,						__A				:	Dict=False		):
     UpperCAmelCase			:				int										=					super()._prepare_for_class(__A,						__A,						return_labels=__A		)

     if return_labels:
       if model_class in get_values(__A		):
         UpperCAmelCase			:				Tuple										=					torch.zeros(
             (self.model_tester.batch_size, self.model_tester.seq_length),						dtype=torch.long,						device=__A		)
     return inputs_dict



   def 			__magic_name__       (				self				:	List[Any]		):
     UpperCAmelCase			:				Optional[int]										=					FunnelModelTester(self		)
     UpperCAmelCase			:				str										=					ConfigTester(self,						config_class=__A		)



   def 			__magic_name__       (				self				:	Dict		):
     self.config_tester.run_common_tests()



   def 			__magic_name__       (				self				:	int		):
     UpperCAmelCase			:				Optional[Any]										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_model(*__A		)



   def 			__magic_name__       (				self				:	Optional[Any]		):
     UpperCAmelCase			:				List[Any]										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_pretraining(*__A		)



   def 			__magic_name__       (				self				:	Dict		):
     UpperCAmelCase			:				str										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_masked_lm(*__A		)



   def 			__magic_name__       (				self				:	str		):
     UpperCAmelCase			:				Any										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_token_classification(*__A		)



   def 			__magic_name__       (				self				:	List[Any]		):
     UpperCAmelCase			:				int										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_question_answering(*__A		)





   def 			__magic_name__       (				self				:	Any,						__A				:	List[Any]		):
     if hasattr(__A,						'''weight'''		) and module.weight is not None:
       module.weight.data.fill_(3		)
     if hasattr(__A,						'''bias'''		) and module.bias is not None:
       module.bias.data.fill_(3		)

     for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
       if hasattr(__A,						__A		) and getattr(__A,						__A		) is not None:
         UpperCAmelCase			:				Optional[int]										=					getattr(__A,						__A		)
         weight.data.fill_(3		)






@require_torch
class    __UpperCAmelCase	(			lowerCamelCase__     ,    unittest.TestCase		):
   UpperCamelCase  	=	False
   UpperCamelCase  	=	False
   UpperCamelCase  	=	(
       (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
   )



   def 			__magic_name__       (				self				:	List[Any]		):
     UpperCAmelCase			:				Optional[int]										=					FunnelModelTester(self,						base=__A		)
     UpperCAmelCase			:				Tuple										=					ConfigTester(self,						config_class=__A		)



   def 			__magic_name__       (				self				:	Union[str, Any]		):
     self.config_tester.run_common_tests()



   def 			__magic_name__       (				self				:	Any		):
     UpperCAmelCase			:				Union[str, Any]										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_base_model(*__A		)



   def 			__magic_name__       (				self				:	Optional[Any]		):
     UpperCAmelCase			:				Union[str, Any]										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_sequence_classification(*__A		)



   def 			__magic_name__       (				self				:	Optional[int]		):
     UpperCAmelCase			:				Tuple										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_multiple_choice(*__A		)



   def 			__magic_name__       (				self				:	Optional[int]		):
     UpperCAmelCase					,		UpperCAmelCase			:				List[str]										=					self.model_tester.prepare_config_and_inputs_for_common()
     UpperCAmelCase			:				List[str]										=					True

     for model_class in self.all_model_classes:
       if model_class.__name__ == "FunnelBaseModel":
         continue
       UpperCAmelCase			:				Any										=					model_class(__A		)
       model.to(__A		)
       model.train()
       UpperCAmelCase			:				str										=					self._prepare_for_class(__A,						__A,						return_labels=__A		)
       UpperCAmelCase			:				str										=					model(**__A		).loss
       loss.backward()





   def 			__magic_name__       (				self				:	str,						__A				:	Any		):
     if hasattr(__A,						'''weight'''		) and module.weight is not None:
       module.weight.data.fill_(3		)
     if hasattr(__A,						'''bias'''		) and module.bias is not None:
       module.bias.data.fill_(3		)

     for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
       if hasattr(__A,						__A		) and getattr(__A,						__A		) is not None:
         UpperCAmelCase			:				Optional[int]										=					getattr(__A,						__A		)
         weight.data.fill_(3		)






@require_torch
@require_sentencepiece
@require_tokenizers
class    __UpperCAmelCase	(			unittest.TestCase		):



   def 			__magic_name__       (				self				:	int		):
     UpperCAmelCase			:				Optional[Any]										=					1_3
     UpperCAmelCase			:				Tuple										=					7
     UpperCAmelCase			:				Optional[Any]										=					torch.arange(0,						batch_size * sequence_length		).long().reshape(__A,						__A		)
     UpperCAmelCase			:				List[Any]										=					[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
     UpperCAmelCase			:				Optional[Any]										=					torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]		)

     UpperCAmelCase			:				Dict										=					FunnelModel.from_pretrained('''sgugger/funnel-random-tiny'''		)
     UpperCAmelCase			:				List[Any]										=					model(__A,						token_type_ids=__A		)[0].abs()

     UpperCAmelCase			:				Union[str, Any]										=					torch.tensor(2_3_4_4.8_3_5_2		)
     UpperCAmelCase			:				List[Any]										=					torch.tensor(0.8_0_5_2		)
     self.assertTrue(torch.allclose(output.sum(),						__A,						atol=1E-4		)		)
     self.assertTrue(torch.allclose(output.mean(),						__A,						atol=1E-4		)		)

     UpperCAmelCase			:				Union[str, Any]										=					torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]		)
     UpperCAmelCase			:				Union[str, Any]										=					model(__A,						attention_mask=__A,						token_type_ids=__A		)[0].abs()

     UpperCAmelCase			:				str										=					torch.tensor(2_3_4_3.8_4_2_5		)
     UpperCAmelCase			:				Optional[Any]										=					torch.tensor(0.8_0_4_9		)
     self.assertTrue(torch.allclose(output.sum(),						__A,						atol=1E-4		)		)
     self.assertTrue(torch.allclose(output.mean(),						__A,						atol=1E-4		)		)





   @slow
   def 			__magic_name__       (				self				:	int		):
     UpperCAmelCase			:				List[Any]										=					FunnelTokenizer.from_pretrained('''huggingface/funnel-small'''		)
     UpperCAmelCase			:				List[str]										=					FunnelModel.from_pretrained('''huggingface/funnel-small'''		)
     UpperCAmelCase			:				List[str]										=					tokenizer('''Hello! I am the Funnel Transformer model.''',						return_tensors='''pt'''		)
     UpperCAmelCase			:				Optional[Any]										=					model(**__A		)[0]

     UpperCAmelCase			:				int										=					torch.tensor(2_3_5.7_2_4_6		)
     UpperCAmelCase			:				Optional[int]										=					torch.tensor(0.0_2_5_6		)
     self.assertTrue(torch.allclose(output.sum(),						__A,						atol=1E-4		)		)
     self.assertTrue(torch.allclose(output.mean(),						__A,						atol=1E-4		)		)





