
import argparse
import json

import gdown
import numpy as np
import torch
from huggingface_hub import hf_hub_download

from transformers import TimesformerConfig, TimesformerForVideoClassification, VideoMAEImageProcessor







def 				UpperCAmelCase_(    a__   ):



    """simple docstring"""



    SCREAMING_SNAKE_CASE				:				Any				      = TimesformerConfig()

    if "large" in model_name:
        SCREAMING_SNAKE_CASE				:				Dict				      = 96

    if "hr" in model_name:
        SCREAMING_SNAKE_CASE				:				Optional[Any]				      = 16
        SCREAMING_SNAKE_CASE				:				str				      = 448

    SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = '''huggingface/label-files'''
    if "k400" in model_name:
        SCREAMING_SNAKE_CASE				:				Optional[Any]				      = 400
        SCREAMING_SNAKE_CASE				:				Tuple				      = '''kinetics400-id2label.json'''
    elif "k600" in model_name:
        SCREAMING_SNAKE_CASE				:				Any				      = 600
        SCREAMING_SNAKE_CASE				:				int				      = '''kinetics600-id2label.json'''
    elif "ssv2" in model_name:
        SCREAMING_SNAKE_CASE				:				Tuple				      = 174
        SCREAMING_SNAKE_CASE				:				int				      = '''something-something-v2-id2label.json'''
    else:
        raise ValueError('''Model name should either contain \'k400\', \'k600\' or \'ssv2\'.'''   )
    SCREAMING_SNAKE_CASE				:				List[str]				      = json.load(open(hf_hub_download(a__ ,						a__ ,						repo_type='''dataset'''   ) ,						'''r'''   )   )
    SCREAMING_SNAKE_CASE				:				List[Any]				      = {int(a__   ): v for k, v in idalabel.items()}
    SCREAMING_SNAKE_CASE				:				str				      = idalabel
    SCREAMING_SNAKE_CASE				:				Optional[Any]				      = {v: k for k, v in idalabel.items()}

    return config







def 				UpperCAmelCase_(    a__   ):



    """simple docstring"""



    if "encoder." in name:
        SCREAMING_SNAKE_CASE				:				Tuple				      = name.replace('''encoder.''' ,						''''''   )
    if "cls_token" in name:
        SCREAMING_SNAKE_CASE				:				List[str]				      = name.replace('''cls_token''' ,						'''timesformer.embeddings.cls_token'''   )
    if "pos_embed" in name:
        SCREAMING_SNAKE_CASE				:				Optional[int]				      = name.replace('''pos_embed''' ,						'''timesformer.embeddings.position_embeddings'''   )
    if "time_embed" in name:
        SCREAMING_SNAKE_CASE				:				Any				      = name.replace('''time_embed''' ,						'''timesformer.embeddings.time_embeddings'''   )
    if "patch_embed.proj" in name:
        SCREAMING_SNAKE_CASE				:				Any				      = name.replace('''patch_embed.proj''' ,						'''timesformer.embeddings.patch_embeddings.projection'''   )
    if "patch_embed.norm" in name:
        SCREAMING_SNAKE_CASE				:				int				      = name.replace('''patch_embed.norm''' ,						'''timesformer.embeddings.norm'''   )
    if "blocks" in name:
        SCREAMING_SNAKE_CASE				:				Any				      = name.replace('''blocks''' ,						'''timesformer.encoder.layer'''   )
    if "attn.proj" in name:
        SCREAMING_SNAKE_CASE				:				int				      = name.replace('''attn.proj''' ,						'''attention.output.dense'''   )
    if "attn" in name and "bias" not in name and "temporal" not in name:
        SCREAMING_SNAKE_CASE				:				int				      = name.replace('''attn''' ,						'''attention.self'''   )
    if "attn" in name and "temporal" not in name:
        SCREAMING_SNAKE_CASE				:				int				      = name.replace('''attn''' ,						'''attention.attention'''   )
    if "temporal_norm1" in name:
        SCREAMING_SNAKE_CASE				:				List[str]				      = name.replace('''temporal_norm1''' ,						'''temporal_layernorm'''   )
    if "temporal_attn.proj" in name:
        SCREAMING_SNAKE_CASE				:				Optional[Any]				      = name.replace('''temporal_attn''' ,						'''temporal_attention.output.dense'''   )
    if "temporal_fc" in name:
        SCREAMING_SNAKE_CASE				:				int				      = name.replace('''temporal_fc''' ,						'''temporal_dense'''   )
    if "norm1" in name and "temporal" not in name:
        SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = name.replace('''norm1''' ,						'''layernorm_before'''   )
    if "norm2" in name:
        SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = name.replace('''norm2''' ,						'''layernorm_after'''   )
    if "mlp.fc1" in name:
        SCREAMING_SNAKE_CASE				:				List[str]				      = name.replace('''mlp.fc1''' ,						'''intermediate.dense'''   )
    if "mlp.fc2" in name:
        SCREAMING_SNAKE_CASE				:				int				      = name.replace('''mlp.fc2''' ,						'''output.dense'''   )
    if "norm.weight" in name and "fc" not in name and "temporal" not in name:
        SCREAMING_SNAKE_CASE				:				Optional[int]				      = name.replace('''norm.weight''' ,						'''timesformer.layernorm.weight'''   )
    if "norm.bias" in name and "fc" not in name and "temporal" not in name:
        SCREAMING_SNAKE_CASE				:				int				      = name.replace('''norm.bias''' ,						'''timesformer.layernorm.bias'''   )
    if "head" in name:
        SCREAMING_SNAKE_CASE				:				Any				      = name.replace('''head''' ,						'''classifier'''   )

    return name







def 				UpperCAmelCase_(    a__ ,						a__   ):



    """simple docstring"""



    for key in orig_state_dict.copy().keys():
        SCREAMING_SNAKE_CASE				:				Optional[Any]				      = orig_state_dict.pop(a__   )

        if key.startswith('''model.'''   ):
            SCREAMING_SNAKE_CASE				:				Dict				      = key.replace('''model.''' ,						''''''   )

        if "qkv" in key:
            SCREAMING_SNAKE_CASE				:				int				      = key.split('''.'''   )
            SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = int(key_split[1]   )
            SCREAMING_SNAKE_CASE				:				int				      = '''timesformer.encoder.layer.'''
            if "temporal" in key:
                SCREAMING_SNAKE_CASE				:				Tuple				      = '''.temporal_attention.attention.qkv.'''
            else:
                SCREAMING_SNAKE_CASE				:				List[Any]				      = '''.attention.attention.qkv.'''
            if "weight" in key:
                SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = val
            else:
                SCREAMING_SNAKE_CASE				:				Optional[int]				      = val
        else:
            SCREAMING_SNAKE_CASE				:				Tuple				      = val

    return orig_state_dict







def 				UpperCAmelCase_(    ):



    """simple docstring"""



    SCREAMING_SNAKE_CASE				:				Optional[int]				      = hf_hub_download(
        repo_id='''hf-internal-testing/spaghetti-video''' ,						filename='''eating_spaghetti.npy''' ,						repo_type='''dataset'''   )
    SCREAMING_SNAKE_CASE				:				int				      = np.load(a__   )
    return list(a__   )







def 				UpperCAmelCase_(    a__ ,						a__ ,						a__ ,						a__   ):



    """simple docstring"""



    SCREAMING_SNAKE_CASE				:				Dict				      = get_timesformer_config(a__   )

    SCREAMING_SNAKE_CASE				:				Tuple				      = TimesformerForVideoClassification(a__   )

    # download original checkpoint, hosted on Google Drive
    SCREAMING_SNAKE_CASE				:				Tuple				      = '''pytorch_model.bin'''
    gdown.cached_download(a__ ,						a__ ,						quiet=a__   )
    SCREAMING_SNAKE_CASE				:				int				      = torch.load(a__ ,						map_location='''cpu'''   )
    if "model" in files:
        SCREAMING_SNAKE_CASE				:				Tuple				      = files['''model''']
    elif "module" in files:
        SCREAMING_SNAKE_CASE				:				Optional[Any]				      = files['''module''']
    else:
        SCREAMING_SNAKE_CASE				:				Any				      = files['''model_state''']
    SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = convert_state_dict(a__ ,						a__   )

    model.load_state_dict(a__   )
    model.eval()

    # verify model on basic input
    SCREAMING_SNAKE_CASE				:				str				      = VideoMAEImageProcessor(image_mean=[0.5, 0.5, 0.5] ,						image_std=[0.5, 0.5, 0.5]   )
    SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = prepare_video()
    SCREAMING_SNAKE_CASE				:				str				      = image_processor(video[:8] ,						return_tensors='''pt'''   )

    SCREAMING_SNAKE_CASE				:				Any				      = model(**a__   )
    SCREAMING_SNAKE_CASE				:				List[str]				      = outputs.logits

    SCREAMING_SNAKE_CASE				:				Optional[Any]				      = [
        # Kinetics-400 checkpoints (hr = high resolution input of 448px instead of 224px)
        '''timesformer-base-finetuned-k400''',
        '''timesformer-large-finetuned-k400''',
        '''timesformer-hr-finetuned-k400''',
        # Kinetics-600 checkpoints (hr = high resolution input of 448px instead of 224px)
        '''timesformer-base-finetuned-k600''',
        '''timesformer-large-finetuned-k600''',
        '''timesformer-hr-finetuned-k600''',
        # Something-Something-v2 checkpoints (hr = high resolution input of 448px instead of 224px)
        '''timesformer-base-finetuned-ssv2''',
        '''timesformer-large-finetuned-ssv2''',
        '''timesformer-hr-finetuned-ssv2''',
    ]

    # NOTE: logits were tested with image_mean and image_std equal to [0.5, 0.5, 0.5] and [0.5, 0.5, 0.5]
    if model_name == "timesformer-base-finetuned-k400":
        SCREAMING_SNAKE_CASE				:				Tuple				      = torch.Size([1, 400]   )
        SCREAMING_SNAKE_CASE				:				int				      = torch.tensor([-0.3_016, -0.7_713, -0.4_205]   )
    elif model_name == "timesformer-base-finetuned-k600":
        SCREAMING_SNAKE_CASE				:				Tuple				      = torch.Size([1, 600]   )
        SCREAMING_SNAKE_CASE				:				str				      = torch.tensor([-0.7_267, -0.7_466, 3.2_404]   )
    elif model_name == "timesformer-base-finetuned-ssv2":
        SCREAMING_SNAKE_CASE				:				Dict				      = torch.Size([1, 174]   )
        SCREAMING_SNAKE_CASE				:				int				      = torch.tensor([-0.9_059, 0.6_433, -3.1_457]   )
    elif model_name == "timesformer-large-finetuned-k400":
        SCREAMING_SNAKE_CASE				:				Any				      = torch.Size([1, 400]   )
        SCREAMING_SNAKE_CASE				:				int				      = torch.tensor([0, 0, 0]   )
    elif model_name == "timesformer-large-finetuned-k600":
        SCREAMING_SNAKE_CASE				:				str				      = torch.Size([1, 600]   )
        SCREAMING_SNAKE_CASE				:				str				      = torch.tensor([0, 0, 0]   )
    elif model_name == "timesformer-large-finetuned-ssv2":
        SCREAMING_SNAKE_CASE				:				Optional[Any]				      = torch.Size([1, 174]   )
        SCREAMING_SNAKE_CASE				:				str				      = torch.tensor([0, 0, 0]   )
    elif model_name == "timesformer-hr-finetuned-k400":
        SCREAMING_SNAKE_CASE				:				List[str]				      = torch.Size([1, 400]   )
        SCREAMING_SNAKE_CASE				:				int				      = torch.tensor([-0.9_617, -3.7_311, -3.7_708]   )
    elif model_name == "timesformer-hr-finetuned-k600":
        SCREAMING_SNAKE_CASE				:				Optional[Any]				      = torch.Size([1, 600]   )
        SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = torch.tensor([2.5_273, 0.7_127, 1.8_848]   )
    elif model_name == "timesformer-hr-finetuned-ssv2":
        SCREAMING_SNAKE_CASE				:				Optional[Any]				      = torch.Size([1, 174]   )
        SCREAMING_SNAKE_CASE				:				str				      = torch.tensor([-3.6_756, -0.7_513, 0.7_180]   )
    else:
        raise ValueError(F"""Model name not supported. Should be one of {model_names}"""   )

    # verify logits
    assert logits.shape == expected_shape
    assert torch.allclose(logits[0, :3] ,						a__ ,						atol=1e-4   )
    print('''Logits ok!'''   )

    if pytorch_dump_folder_path is not None:
        print(F"""Saving model and image processor to {pytorch_dump_folder_path}"""   )
        image_processor.save_pretrained(a__   )
        model.save_pretrained(a__   )

    if push_to_hub:
        print('''Pushing to the hub...'''   )
        model.push_to_hub(F"""fcakyon/{model_name}"""   )


if __name__ == "__main__":
      a__				:      Optional[int]    		=     argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          '''--checkpoint_url''',
          default='''https://drive.google.com/u/1/uc?id=17yvuYp9L4mn-HpIcK5Zo6K3UoOy1kA5l&export=download''',
          type=str,
          help=(
              '''URL of the original PyTorch checkpoint (on Google Drive) you\'d like to convert. Should be a direct'''
              ''' download link.'''
          ),
      )
      parser.add_argument(
          '''--pytorch_dump_folder_path''',
          default='''''',
          type=str,
          help='''Path to the output PyTorch model directory.''',
      )
      parser.add_argument('''--model_name''', default='''timesformer-base-finetuned-k400''', type=str, help='''Name of the model.''')
      parser.add_argument(
          '''--push_to_hub''', action='''store_true''', help='''Whether or not to push the converted model to the 🤗 hub.'''
      )

      a__				:      str    		=     parser.parse_args()
      convert_timesformer_checkpoint(
          args.checkpoint_url, args.pytorch_dump_folder_path, args.model_name, args.push_to_hub
      )


import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          MODEL_FOR_PRETRAINING_MAPPING,
          FunnelBaseModel,
          FunnelForMaskedLM,
          FunnelForMultipleChoice,
          FunnelForPreTraining,
          FunnelForQuestionAnswering,
          FunnelForSequenceClassification,
          FunnelForTokenClassification,
          FunnelModel,
      )






class     a_       :



       """simple docstring"""






       def __init__(		self				,       _lowerCamelCase				,       _lowerCamelCase=13				,       _lowerCamelCase=7				,       _lowerCamelCase=True				,       _lowerCamelCase=True				,       _lowerCamelCase=True				,       _lowerCamelCase=True				,       _lowerCamelCase=99				,       _lowerCamelCase=[1, 1, 2]				,       _lowerCamelCase=1				,       _lowerCamelCase=32				,       _lowerCamelCase=4				,       _lowerCamelCase=8				,       _lowerCamelCase=37				,       _lowerCamelCase="gelu_new"				,       _lowerCamelCase=0.1				,       _lowerCamelCase=0.1				,       _lowerCamelCase=0.0				,       _lowerCamelCase=512				,       _lowerCamelCase=3				,       _lowerCamelCase=0.0_2				,       _lowerCamelCase=3				,       _lowerCamelCase=4				,       _lowerCamelCase=None				,       _lowerCamelCase=False				,       )					->List[str]:
           SCREAMING_SNAKE_CASE				:				Any				      = parent
           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = batch_size
           SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = seq_length
           SCREAMING_SNAKE_CASE				:				Dict				      = is_training
           SCREAMING_SNAKE_CASE				:				int				      = use_input_mask
           SCREAMING_SNAKE_CASE				:				Optional[int]				      = use_token_type_ids
           SCREAMING_SNAKE_CASE				:				Tuple				      = use_labels
           SCREAMING_SNAKE_CASE				:				List[str]				      = vocab_size
           SCREAMING_SNAKE_CASE				:				Tuple				      = block_sizes
           SCREAMING_SNAKE_CASE				:				str				      = num_decoder_layers
           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = d_model
           SCREAMING_SNAKE_CASE				:				str				      = n_head
           SCREAMING_SNAKE_CASE				:				List[str]				      = d_head
           SCREAMING_SNAKE_CASE				:				Dict				      = d_inner
           SCREAMING_SNAKE_CASE				:				Any				      = hidden_act
           SCREAMING_SNAKE_CASE				:				Dict				      = hidden_dropout
           SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = attention_dropout
           SCREAMING_SNAKE_CASE				:				List[Any]				      = activation_dropout
           SCREAMING_SNAKE_CASE				:				str				      = max_position_embeddings
           SCREAMING_SNAKE_CASE				:				Optional[int]				      = type_vocab_size
           SCREAMING_SNAKE_CASE				:				Dict				      = 2
           SCREAMING_SNAKE_CASE				:				Tuple				      = num_labels
           SCREAMING_SNAKE_CASE				:				str				      = num_choices
           SCREAMING_SNAKE_CASE				:				Tuple				      = scope
           SCREAMING_SNAKE_CASE				:				List[str]				      = initializer_std

           # Used in the tests to check the size of the first attention layer
           SCREAMING_SNAKE_CASE				:				str				      = n_head
           # Used in the tests to check the size of the first hidden state
           SCREAMING_SNAKE_CASE				:				Tuple				      = self.d_model
           # Used in the tests to check the number of output hidden states/attentions
           SCREAMING_SNAKE_CASE				:				str				      = sum(self.block_sizes   ) + (0 if base else self.num_decoder_layers)
           # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
           # the last hidden state of the first block (which is the first hidden state of the decoder).
           if not base:
               SCREAMING_SNAKE_CASE				:				Tuple				      = self.num_hidden_layers + 2






       def 	__lowerCAmelCase     (		self   )					->Union[str, Any]:
           SCREAMING_SNAKE_CASE				:				str				      = ids_tensor([self.batch_size, self.seq_length]				,       self.vocab_size   )

           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = None
           if self.use_input_mask:
               SCREAMING_SNAKE_CASE				:				Optional[Any]				      = ids_tensor([self.batch_size, self.seq_length]				,       vocab_size=2   )

           SCREAMING_SNAKE_CASE				:				List[str]				      = None
           if self.use_token_type_ids:
               SCREAMING_SNAKE_CASE				:				Dict				      = ids_tensor([self.batch_size, self.seq_length]				,       self.type_vocab_size   )

           SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = None
           SCREAMING_SNAKE_CASE				:				Tuple				      = None
           SCREAMING_SNAKE_CASE				:				List[str]				      = None
           if self.use_labels:
               SCREAMING_SNAKE_CASE				:				Tuple				      = ids_tensor([self.batch_size]				,       self.type_sequence_label_size   )
               SCREAMING_SNAKE_CASE				:				str				      = ids_tensor([self.batch_size, self.seq_length]				,       self.num_labels   )
               SCREAMING_SNAKE_CASE				:				Any				      = ids_tensor([self.batch_size]				,       self.num_choices   )
               SCREAMING_SNAKE_CASE				:				Optional[Any]				      = ids_tensor([self.batch_size, self.seq_length]				,       1   )

           SCREAMING_SNAKE_CASE				:				Tuple				      = self.get_config()

           return (
               config,
               input_ids,
               token_type_ids,
               input_mask,
               sequence_labels,
               token_labels,
               choice_labels,
               fake_token_labels,
           )






       def 	__lowerCAmelCase     (		self   )					->str:
           return FunnelConfig(
               vocab_size=self.vocab_size				,       block_sizes=self.block_sizes				,       num_decoder_layers=self.num_decoder_layers				,       d_model=self.d_model				,       n_head=self.n_head				,       d_head=self.d_head				,       d_inner=self.d_inner				,       hidden_act=self.hidden_act				,       hidden_dropout=self.hidden_dropout				,       attention_dropout=self.attention_dropout				,       activation_dropout=self.activation_dropout				,       max_position_embeddings=self.max_position_embeddings				,       type_vocab_size=self.type_vocab_size				,       initializer_std=self.initializer_std				,       )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       )					->Optional[Any]:
           SCREAMING_SNAKE_CASE				:				List[Any]				      = FunnelModel(config=_lowerCamelCase   )
           model.to(_lowerCamelCase   )
           model.eval()
           SCREAMING_SNAKE_CASE				:				Dict				      = model(_lowerCamelCase				,       attention_mask=_lowerCamelCase				,       token_type_ids=_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				List[str]				      = model(_lowerCamelCase				,       token_type_ids=_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				Optional[int]				      = model(_lowerCamelCase   )
           self.parent.assertEqual(result.last_hidden_state.shape				,       (self.batch_size, self.seq_length, self.d_model)   )

           SCREAMING_SNAKE_CASE				:				int				      = False
           SCREAMING_SNAKE_CASE				:				List[Any]				      = model(_lowerCamelCase   )
           self.parent.assertEqual(result.last_hidden_state.shape				,       (self.batch_size, self.seq_length, self.d_model)   )

           SCREAMING_SNAKE_CASE				:				str				      = False
           SCREAMING_SNAKE_CASE				:				List[str]				      = model(_lowerCamelCase   )
           self.parent.assertEqual(result.last_hidden_state.shape				,       (self.batch_size, self.seq_length, self.d_model)   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       )					->Any:
           SCREAMING_SNAKE_CASE				:				Optional[int]				      = FunnelBaseModel(config=_lowerCamelCase   )
           model.to(_lowerCamelCase   )
           model.eval()
           SCREAMING_SNAKE_CASE				:				Dict				      = model(_lowerCamelCase				,       attention_mask=_lowerCamelCase				,       token_type_ids=_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				str				      = model(_lowerCamelCase				,       token_type_ids=_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = model(_lowerCamelCase   )
           self.parent.assertEqual(result.last_hidden_state.shape				,       (self.batch_size, 2, self.d_model)   )

           SCREAMING_SNAKE_CASE				:				Optional[int]				      = False
           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = model(_lowerCamelCase   )
           self.parent.assertEqual(result.last_hidden_state.shape				,       (self.batch_size, 3, self.d_model)   )

           SCREAMING_SNAKE_CASE				:				List[Any]				      = False
           SCREAMING_SNAKE_CASE				:				Tuple				      = model(_lowerCamelCase   )
           self.parent.assertEqual(result.last_hidden_state.shape				,       (self.batch_size, 2, self.d_model)   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       )					->int:
           SCREAMING_SNAKE_CASE				:				int				      = self.num_labels
           SCREAMING_SNAKE_CASE				:				int				      = FunnelForPreTraining(config=_lowerCamelCase   )
           model.to(_lowerCamelCase   )
           model.eval()
           SCREAMING_SNAKE_CASE				:				str				      = model(_lowerCamelCase				,       attention_mask=_lowerCamelCase				,       token_type_ids=_lowerCamelCase				,       labels=_lowerCamelCase   )
           self.parent.assertEqual(result.logits.shape				,       (self.batch_size, self.seq_length)   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       )					->List[str]:
           SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = FunnelForMaskedLM(config=_lowerCamelCase   )
           model.to(_lowerCamelCase   )
           model.eval()
           SCREAMING_SNAKE_CASE				:				Optional[int]				      = model(_lowerCamelCase				,       attention_mask=_lowerCamelCase				,       token_type_ids=_lowerCamelCase				,       labels=_lowerCamelCase   )
           self.parent.assertEqual(result.logits.shape				,       (self.batch_size, self.seq_length, self.vocab_size)   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       )					->List[str]:
           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = self.num_labels
           SCREAMING_SNAKE_CASE				:				Any				      = FunnelForSequenceClassification(_lowerCamelCase   )
           model.to(_lowerCamelCase   )
           model.eval()
           SCREAMING_SNAKE_CASE				:				int				      = model(_lowerCamelCase				,       attention_mask=_lowerCamelCase				,       token_type_ids=_lowerCamelCase				,       labels=_lowerCamelCase   )
           self.parent.assertEqual(result.logits.shape				,       (self.batch_size, self.num_labels)   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       )					->str:
           SCREAMING_SNAKE_CASE				:				Tuple				      = self.num_choices
           SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = FunnelForMultipleChoice(config=_lowerCamelCase   )
           model.to(_lowerCamelCase   )
           model.eval()
           SCREAMING_SNAKE_CASE				:				Tuple				      = input_ids.unsqueeze(1   ).expand(-1				,       self.num_choices				,       -1   ).contiguous()
           SCREAMING_SNAKE_CASE				:				Dict				      = token_type_ids.unsqueeze(1   ).expand(-1				,       self.num_choices				,       -1   ).contiguous()
           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = input_mask.unsqueeze(1   ).expand(-1				,       self.num_choices				,       -1   ).contiguous()
           SCREAMING_SNAKE_CASE				:				str				      = model(
               _lowerCamelCase				,       attention_mask=_lowerCamelCase				,       token_type_ids=_lowerCamelCase				,       labels=_lowerCamelCase				,       )
           self.parent.assertEqual(result.logits.shape				,       (self.batch_size, self.num_choices)   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       )					->Tuple:
           SCREAMING_SNAKE_CASE				:				str				      = self.num_labels
           SCREAMING_SNAKE_CASE				:				int				      = FunnelForTokenClassification(config=_lowerCamelCase   )
           model.to(_lowerCamelCase   )
           model.eval()
           SCREAMING_SNAKE_CASE				:				Dict				      = model(_lowerCamelCase				,       attention_mask=_lowerCamelCase				,       token_type_ids=_lowerCamelCase				,       labels=_lowerCamelCase   )
           self.parent.assertEqual(result.logits.shape				,       (self.batch_size, self.seq_length, self.num_labels)   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       )					->Tuple:
           SCREAMING_SNAKE_CASE				:				List[str]				      = FunnelForQuestionAnswering(config=_lowerCamelCase   )
           model.to(_lowerCamelCase   )
           model.eval()
           SCREAMING_SNAKE_CASE				:				int				      = model(
               _lowerCamelCase				,       attention_mask=_lowerCamelCase				,       token_type_ids=_lowerCamelCase				,       start_positions=_lowerCamelCase				,       end_positions=_lowerCamelCase				,       )
           self.parent.assertEqual(result.start_logits.shape				,       (self.batch_size, self.seq_length)   )
           self.parent.assertEqual(result.end_logits.shape				,       (self.batch_size, self.seq_length)   )






       def 	__lowerCAmelCase     (		self   )					->str:
           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = self.prepare_config_and_inputs()
           (
               (
               SCREAMING_SNAKE_CASE
           )      ,							(
               SCREAMING_SNAKE_CASE
           )      ,							(
               SCREAMING_SNAKE_CASE
           )      ,							(
               SCREAMING_SNAKE_CASE
           )      ,							(
               SCREAMING_SNAKE_CASE
           )      ,							(
               SCREAMING_SNAKE_CASE
           )      ,							(
               SCREAMING_SNAKE_CASE
           )      ,							(
               SCREAMING_SNAKE_CASE
           )      ,							
           )				:				Optional[int]				      = config_and_inputs
           SCREAMING_SNAKE_CASE				:				int				      = {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
           return config, inputs_dict






@require_torch
class     a_       (			a__				,    a__				,    unittest.TestCase							):



       """simple docstring"""
       __SCREAMING_SNAKE_CASE       :					str						=  False
       __SCREAMING_SNAKE_CASE       :					List[str]						=  False
       __SCREAMING_SNAKE_CASE       :					Tuple						=  (
           (
               FunnelModel,
               FunnelForMaskedLM,
               FunnelForPreTraining,
               FunnelForQuestionAnswering,
               FunnelForTokenClassification,
           )
           if is_torch_available()
           else ()
       )
       __SCREAMING_SNAKE_CASE       :					Tuple						=  (
           {
               'feature-extraction': (FunnelBaseModel, FunnelModel),
               'fill-mask': FunnelForMaskedLM,
               'question-answering': FunnelForQuestionAnswering,
               'text-classification': FunnelForSequenceClassification,
               'token-classification': FunnelForTokenClassification,
               'zero-shot': FunnelForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase=False   )					->str:
           SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = super()._prepare_for_class(_lowerCamelCase				,       _lowerCamelCase				,       return_labels=_lowerCamelCase   )

           if return_labels:
               if model_class in get_values(_lowerCamelCase   ):
                   SCREAMING_SNAKE_CASE				:				List[Any]				      = torch.zeros(
                       (self.model_tester.batch_size, self.model_tester.seq_length)				,       dtype=torch.long				,       device=_lowerCamelCase   )
           return inputs_dict






       def 	__lowerCAmelCase     (		self   )					->int:
           SCREAMING_SNAKE_CASE				:				List[Any]				      = FunnelModelTester(self   )
           SCREAMING_SNAKE_CASE				:				Any				      = ConfigTester(self				,       config_class=_lowerCamelCase   )






       def 	__lowerCAmelCase     (		self   )					->int:
           self.config_tester.run_common_tests()






       def 	__lowerCAmelCase     (		self   )					->Optional[Any]:
           SCREAMING_SNAKE_CASE				:				List[Any]				      = self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*_lowerCamelCase   )






       def 	__lowerCAmelCase     (		self   )					->int:
           SCREAMING_SNAKE_CASE				:				Any				      = self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_pretraining(*_lowerCamelCase   )






       def 	__lowerCAmelCase     (		self   )					->Dict:
           SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_masked_lm(*_lowerCamelCase   )






       def 	__lowerCAmelCase     (		self   )					->Tuple:
           SCREAMING_SNAKE_CASE				:				Any				      = self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_token_classification(*_lowerCamelCase   )






       def 	__lowerCAmelCase     (		self   )					->Tuple:
           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_question_answering(*_lowerCamelCase   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase   )					->List[Any]:
           if hasattr(_lowerCamelCase				,       '''weight'''   ) and module.weight is not None:
               module.weight.data.fill_(3   )
           if hasattr(_lowerCamelCase				,       '''bias'''   ) and module.bias is not None:
               module.bias.data.fill_(3   )

           for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
               if hasattr(_lowerCamelCase				,       _lowerCamelCase   ) and getattr(_lowerCamelCase				,       _lowerCamelCase   ) is not None:
                   SCREAMING_SNAKE_CASE				:				int				      = getattr(_lowerCamelCase				,       _lowerCamelCase   )
                   weight.data.fill_(3   )






@require_torch
class     a_       (			a__				,    unittest.TestCase							):



       """simple docstring"""
       __SCREAMING_SNAKE_CASE       :					List[str]						=  False
       __SCREAMING_SNAKE_CASE       :					Any						=  False
       __SCREAMING_SNAKE_CASE       :					Dict						=  (
           (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
       )






       def 	__lowerCAmelCase     (		self   )					->Dict:
           SCREAMING_SNAKE_CASE				:				str				      = FunnelModelTester(self				,       base=_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				Dict				      = ConfigTester(self				,       config_class=_lowerCamelCase   )






       def 	__lowerCAmelCase     (		self   )					->Union[str, Any]:
           self.config_tester.run_common_tests()






       def 	__lowerCAmelCase     (		self   )					->Any:
           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_base_model(*_lowerCamelCase   )






       def 	__lowerCAmelCase     (		self   )					->Tuple:
           SCREAMING_SNAKE_CASE				:				Tuple				      = self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_sequence_classification(*_lowerCamelCase   )






       def 	__lowerCAmelCase     (		self   )					->Dict:
           SCREAMING_SNAKE_CASE				:				Any				      = self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_multiple_choice(*_lowerCamelCase   )






       def 	__lowerCAmelCase     (		self   )					->List[str]:
           SCREAMING_SNAKE_CASE      ,							SCREAMING_SNAKE_CASE				:				Tuple				      = self.model_tester.prepare_config_and_inputs_for_common()
           SCREAMING_SNAKE_CASE				:				int				      = True

           for model_class in self.all_model_classes:
               if model_class.__name__ == "FunnelBaseModel":
                   continue
               SCREAMING_SNAKE_CASE				:				List[str]				      = model_class(_lowerCamelCase   )
               model.to(_lowerCamelCase   )
               model.train()
               SCREAMING_SNAKE_CASE				:				List[str]				      = self._prepare_for_class(_lowerCamelCase				,       _lowerCamelCase				,       return_labels=_lowerCamelCase   )
               SCREAMING_SNAKE_CASE				:				int				      = model(**_lowerCamelCase   ).loss
               loss.backward()






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase   )					->Union[str, Any]:
           if hasattr(_lowerCamelCase				,       '''weight'''   ) and module.weight is not None:
               module.weight.data.fill_(3   )
           if hasattr(_lowerCamelCase				,       '''bias'''   ) and module.bias is not None:
               module.bias.data.fill_(3   )

           for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
               if hasattr(_lowerCamelCase				,       _lowerCamelCase   ) and getattr(_lowerCamelCase				,       _lowerCamelCase   ) is not None:
                   SCREAMING_SNAKE_CASE				:				List[str]				      = getattr(_lowerCamelCase				,       _lowerCamelCase   )
                   weight.data.fill_(3   )






@require_torch
@require_sentencepiece
@require_tokenizers
class     a_       (			unittest.TestCase							):



       """simple docstring"""






       def 	__lowerCAmelCase     (		self   )					->Tuple:
           SCREAMING_SNAKE_CASE				:				List[Any]				      = 13
           SCREAMING_SNAKE_CASE				:				List[str]				      = 7
           SCREAMING_SNAKE_CASE				:				Any				      = torch.arange(0				,       batch_size * sequence_length   ).long().reshape(_lowerCamelCase				,       _lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				List[Any]				      = [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
           SCREAMING_SNAKE_CASE				:				Tuple				      = torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]   )

           SCREAMING_SNAKE_CASE				:				Tuple				      = FunnelModel.from_pretrained('''sgugger/funnel-random-tiny'''   )
           SCREAMING_SNAKE_CASE				:				Any				      = model(_lowerCamelCase				,       token_type_ids=_lowerCamelCase   )[0].abs()

           SCREAMING_SNAKE_CASE				:				str				      = torch.tensor(2_3_4_4.8_3_5_2   )
           SCREAMING_SNAKE_CASE				:				Any				      = torch.tensor(0.8_0_5_2   )
           self.assertTrue(torch.allclose(output.sum()				,       _lowerCamelCase				,       atol=1e-4   )   )
           self.assertTrue(torch.allclose(output.mean()				,       _lowerCamelCase				,       atol=1e-4   )   )

           SCREAMING_SNAKE_CASE				:				List[Any]				      = torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]   )
           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = model(_lowerCamelCase				,       attention_mask=_lowerCamelCase				,       token_type_ids=_lowerCamelCase   )[0].abs()

           SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = torch.tensor(2_3_4_3.8_4_2_5   )
           SCREAMING_SNAKE_CASE				:				Dict				      = torch.tensor(0.8_0_4_9   )
           self.assertTrue(torch.allclose(output.sum()				,       _lowerCamelCase				,       atol=1e-4   )   )
           self.assertTrue(torch.allclose(output.mean()				,       _lowerCamelCase				,       atol=1e-4   )   )






       @slow
       def 	__lowerCAmelCase     (		self   )					->Optional[Any]:
           SCREAMING_SNAKE_CASE				:				str				      = FunnelTokenizer.from_pretrained('''huggingface/funnel-small'''   )
           SCREAMING_SNAKE_CASE				:				Any				      = FunnelModel.from_pretrained('''huggingface/funnel-small'''   )
           SCREAMING_SNAKE_CASE				:				Tuple				      = tokenizer('''Hello! I am the Funnel Transformer model.'''				,       return_tensors='''pt'''   )
           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = model(**_lowerCamelCase   )[0]

           SCREAMING_SNAKE_CASE				:				Dict				      = torch.tensor(2_3_5.7_2_4_6   )
           SCREAMING_SNAKE_CASE				:				Dict				      = torch.tensor(0.0_2_5_6   )
           self.assertTrue(torch.allclose(output.sum()				,       _lowerCamelCase				,       atol=1e-4   )   )
           self.assertTrue(torch.allclose(output.mean()				,       _lowerCamelCase				,       atol=1e-4   )   )
