






"""simple docstring"""





import argparse

import torch

from diffusers import UNetaDConditionModel






def 							A(    snake_case_		,  snake_case_		,  snake_case_		,  snake_case_=None		,  snake_case_=None		,  snake_case_=None						):
       """simple docstring"""





       assert isinstance(snake_case_		,  snake_case_						), "Paths should be a list of dicts containing 'old' and 'new' keys."

       # Splits the attention layers into three variables.
       if attention_paths_to_split is not None:
              for path, path_map in attention_paths_to_split.items():
                     lowercase__:							str         =     old_checkpoint[path]
                     lowercase__:							int         =     old_tensor.shape[0] // 3

                     lowercase__:							Any         =     (-1, channels) if len(old_tensor.shape						) == 3 else (-1)

                     lowercase__:							Optional[Any]         =     old_tensor.shape[0] // config["num_head_channels"] // 3

                     lowercase__:							Any         =     old_tensor.reshape((num_heads, 3 * channels // num_heads) + old_tensor.shape[1:]						)
                     lowercase__    ,      lowercase__    ,      lowercase__:							Dict         =     old_tensor.split(channels // num_heads		,  dim=1						)

                     lowercase__:							int         =     query.reshape(snake_case_						)
                     lowercase__:							Union[str, Any]         =     key.reshape(snake_case_						)
                     lowercase__:							List[Any]         =     value.reshape(snake_case_						)

       for path in paths:
              lowercase__:							Union[str, Any]         =     path["new"]

              # These have already been assigned
              if attention_paths_to_split is not None and new_path in attention_paths_to_split:
                     continue

              if additional_replacements is not None:
                     for replacement in additional_replacements:
                            lowercase__:							Optional[int]         =     new_path.replace(replacement["old"]		,  replacement["new"]						)

        # proj_attn.weight has to be converted from conv 1D to linear
              lowercase__:							Any         =     old_checkpoint[path["old"]]
              lowercase__:							Optional[int]         =     ["proj_attn.weight"]
              lowercase__:							Any         =     ["proj_out.weight", "proj_in.weight"]
              if any(k in new_path for k in names						):
                     lowercase__:							Any         =     weight[:, :, 0]
              elif any(k in new_path for k in names_a						) and len(weight.shape						) > 2 and ".attentions." not in new_path:
                     lowercase__:							Union[str, Any]         =     weight[:, :, 0]
              else:
                     lowercase__:							List[Any]         =     weight






def 							A(    snake_case_		,  snake_case_=0						):
       """simple docstring"""





       lowercase__:							List[str]         =     []
       for old_item in old_list:
              lowercase__:							Optional[int]         =     old_item

              #         new_item = new_item.replace('norm.weight', 'group_norm.weight')
              #         new_item = new_item.replace('norm.bias', 'group_norm.bias')

              #         new_item = new_item.replace('proj_out.weight', 'proj_attn.weight')
              #         new_item = new_item.replace('proj_out.bias', 'proj_attn.bias')

              #         new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments)

              mapping.append({"old": old_item, "new": new_item}						)

       return mapping






def 							A(    snake_case_		,  snake_case_=1						):
       """simple docstring"""





       if n_shave_prefix_segments >= 0:
              return ".".join(path.split("."						)[n_shave_prefix_segments:]						)
       else:
              return ".".join(path.split("."						)[:n_shave_prefix_segments]						)






def 							A(    snake_case_		,  snake_case_=0						):
       """simple docstring"""





       lowercase__:							Dict         =     []
       for old_item in old_list:
              mapping.append({"old": old_item, "new": old_item}						)

       return mapping






def 							A(    snake_case_		,  snake_case_=0						):
       """simple docstring"""





       lowercase__:							str         =     []
       for old_item in old_list:
              lowercase__:							Union[str, Any]         =     old_item.replace("in_layers.0"		,  "norm1"						)
              lowercase__:							Optional[Any]         =     new_item.replace("in_layers.2"		,  "conv1"						)

              lowercase__:							Optional[Any]         =     new_item.replace("out_layers.0"		,  "norm2"						)
              lowercase__:							List[str]         =     new_item.replace("out_layers.3"		,  "conv2"						)

              lowercase__:							Optional[int]         =     new_item.replace("emb_layers.1"		,  "time_emb_proj"						)
              lowercase__:							Tuple         =     new_item.replace("skip_connection"		,  "conv_shortcut"						)

              lowercase__:							List[str]         =     shave_segments(snake_case_		,  n_shave_prefix_segments=snake_case_						)

              if "temopral_conv" not in old_item:
                     mapping.append({"old": old_item, "new": new_item}						)

       return mapping






def 							A(    snake_case_		,  snake_case_		,  snake_case_=None		,  snake_case_=False						):
       """simple docstring"""





       lowercase__:							List[Any]         =     {}
       lowercase__:							Optional[Any]         =     list(checkpoint.keys()						)

       lowercase__:							int         =     "model.diffusion_model."

       # at least a 100 parameters have to start with `model_ema` in order for the checkpoint to be EMA
       if sum(k.startswith("model_ema"						) for k in keys						) > 100 and extract_ema:
              print(F"""Checkpoint {path} has both EMA and non-EMA weights."""						)
              print(
                  "In this conversion only the EMA weights are extracted. If you want to instead extract the non-EMA"
                  " weights (useful to continue fine-tuning), please make sure to remove the `--extract_ema` flag."						)
              for key in keys:
                     if key.startswith("model.diffusion_model"						):
                            lowercase__:							Optional[int]         =     "model_ema." + "".join(key.split("."						)[1:]						)
                            lowercase__:							Any         =     checkpoint.pop(snake_case_						)
       else:
              if sum(k.startswith("model_ema"						) for k in keys						) > 100:
                     print(
                         "In this conversion only the non-EMA weights are extracted. If you want to instead extract the EMA"
                         " weights (usually better for inference), please make sure to add the `--extract_ema` flag."						)

              for key in keys:
                     lowercase__:							Optional[int]         =     checkpoint.pop(snake_case_						)

       lowercase__:							List[str]         =     {}

       lowercase__:							str         =     unet_state_dict["time_embed.0.weight"]
       lowercase__:							List[Any]         =     unet_state_dict["time_embed.0.bias"]
       lowercase__:							Tuple         =     unet_state_dict["time_embed.2.weight"]
       lowercase__:							Union[str, Any]         =     unet_state_dict["time_embed.2.bias"]

       if config["class_embed_type"] is None:
              # No parameters to port
              ...
       elif config["class_embed_type"] == "timestep" or config["class_embed_type"] == "projection":
              lowercase__:							int         =     unet_state_dict["label_emb.0.0.weight"]
              lowercase__:							Tuple         =     unet_state_dict["label_emb.0.0.bias"]
              lowercase__:							List[str]         =     unet_state_dict["label_emb.0.2.weight"]
              lowercase__:							str         =     unet_state_dict["label_emb.0.2.bias"]
       else:
              raise NotImplementedError(F"""Not implemented `class_embed_type`: {config['class_embed_type']}"""						)

       lowercase__:							str         =     unet_state_dict["input_blocks.0.0.weight"]
       lowercase__:							Optional[int]         =     unet_state_dict["input_blocks.0.0.bias"]

       lowercase__:							List[str]         =     [v for v in unet_state_dict if v.startswith("input_blocks.0.1"						)]
       lowercase__:							Optional[int]         =     renew_attention_paths(snake_case_						)
       lowercase__:							List[str]         =     {"old": "input_blocks.0.1", "new": "transformer_in"}
       assign_to_checkpoint(snake_case_		,  snake_case_		,  snake_case_		,  additional_replacements=[meta_path]		,  config=snake_case_						)

       lowercase__:							Tuple         =     unet_state_dict["out.0.weight"]
       lowercase__:							List[str]         =     unet_state_dict["out.0.bias"]
       lowercase__:							Union[str, Any]         =     unet_state_dict["out.2.weight"]
       lowercase__:							Tuple         =     unet_state_dict["out.2.bias"]

       # Retrieves the keys for the input blocks only
       lowercase__:							Tuple         =     len({".".join(layer.split("."						)[:2]						) for layer in unet_state_dict if "input_blocks" in layer}						)
       lowercase__:							List[Any]         =     {
           layer_id: [key for key in unet_state_dict if F"""input_blocks.{layer_id}""" in key]
           for layer_id in range(snake_case_						)
       }

       # Retrieves the keys for the middle blocks only
       lowercase__:							List[Any]         =     len({".".join(layer.split("."						)[:2]						) for layer in unet_state_dict if "middle_block" in layer}						)
       lowercase__:							Union[str, Any]         =     {
           layer_id: [key for key in unet_state_dict if F"""middle_block.{layer_id}""" in key]
           for layer_id in range(snake_case_						)
       }

       # Retrieves the keys for the output blocks only
       lowercase__:							Tuple         =     len({".".join(layer.split("."						)[:2]						) for layer in unet_state_dict if "output_blocks" in layer}						)
       lowercase__:							List[str]         =     {
           layer_id: [key for key in unet_state_dict if F"""output_blocks.{layer_id}""" in key]
           for layer_id in range(snake_case_						)
       }

       for i in range(1		,  snake_case_						):
              lowercase__:							List[Any]         =     (i - 1) // (config["layers_per_block"] + 1)
              lowercase__:							int         =     (i - 1) % (config["layers_per_block"] + 1)

              lowercase__:							Optional[int]         =     [
                  key for key in input_blocks[i] if F"""input_blocks.{i}.0""" in key and F"""input_blocks.{i}.0.op""" not in key
              ]
              lowercase__:							Tuple         =     [key for key in input_blocks[i] if F"""input_blocks.{i}.1""" in key]
              lowercase__:							int         =     [key for key in input_blocks[i] if F"""input_blocks.{i}.2""" in key]

              if F"""input_blocks.{i}.op.weight""" in unet_state_dict:
                     lowercase__:							Optional[int]         =     unet_state_dict.pop(
                         F"""input_blocks.{i}.op.weight"""						)
                     lowercase__:							Optional[int]         =     unet_state_dict.pop(
                         F"""input_blocks.{i}.op.bias"""						)

              lowercase__:							Optional[Any]         =     renew_resnet_paths(snake_case_						)
              lowercase__:							Optional[int]         =     {"old": F"""input_blocks.{i}.0""", "new": F"""down_blocks.{block_id}.resnets.{layer_in_block_id}"""}
              assign_to_checkpoint(
                  snake_case_		,  snake_case_		,  snake_case_		,  additional_replacements=[meta_path]		,  config=snake_case_						)

              lowercase__:							Union[str, Any]         =     [key for key in resnets if "temopral_conv" in key]
              lowercase__:							List[str]         =     renew_temp_conv_paths(snake_case_						)
              lowercase__:							Tuple         =     {
                  "old": F"""input_blocks.{i}.0.temopral_conv""",
                  "new": F"""down_blocks.{block_id}.temp_convs.{layer_in_block_id}""",
              }
              assign_to_checkpoint(
                  snake_case_		,  snake_case_		,  snake_case_		,  additional_replacements=[meta_path]		,  config=snake_case_						)

              if len(snake_case_						):
                     lowercase__:							Optional[Any]         =     renew_attention_paths(snake_case_						)
                     lowercase__:							Optional[Any]         =     {"old": F"""input_blocks.{i}.1""", "new": F"""down_blocks.{block_id}.attentions.{layer_in_block_id}"""}
                     assign_to_checkpoint(
                         snake_case_		,  snake_case_		,  snake_case_		,  additional_replacements=[meta_path]		,  config=snake_case_						)

              if len(snake_case_						):
                     lowercase__:							Dict         =     renew_attention_paths(snake_case_						)
                     lowercase__:							str         =     {
                         "old": F"""input_blocks.{i}.2""",
                         "new": F"""down_blocks.{block_id}.temp_attentions.{layer_in_block_id}""",
                     }
                     assign_to_checkpoint(
                         snake_case_		,  snake_case_		,  snake_case_		,  additional_replacements=[meta_path]		,  config=snake_case_						)

       lowercase__:							List[str]         =     middle_blocks[0]
       lowercase__:							str         =     [key for key in resnet_a if "temopral_conv" in key]
       lowercase__:							List[Any]         =     middle_blocks[1]
       lowercase__:							List[str]         =     middle_blocks[2]
       lowercase__:							int         =     middle_blocks[3]
       lowercase__:							Tuple         =     [key for key in resnet_a if "temopral_conv" in key]

       lowercase__:							Optional[Any]         =     renew_resnet_paths(snake_case_						)
       lowercase__:							List[str]         =     {"old": "middle_block.0", "new": "mid_block.resnets.0"}
       assign_to_checkpoint(
           snake_case_		,  snake_case_		,  snake_case_		,  config=snake_case_		,  additional_replacements=[meta_path]						)

       lowercase__:							Tuple         =     renew_temp_conv_paths(snake_case_						)
       lowercase__:							List[Any]         =     {"old": "middle_block.0.temopral_conv", "new": "mid_block.temp_convs.0"}
       assign_to_checkpoint(
           snake_case_		,  snake_case_		,  snake_case_		,  config=snake_case_		,  additional_replacements=[meta_path]						)

       lowercase__:							Tuple         =     renew_resnet_paths(snake_case_						)
       lowercase__:							Any         =     {"old": "middle_block.3", "new": "mid_block.resnets.1"}
       assign_to_checkpoint(
           snake_case_		,  snake_case_		,  snake_case_		,  config=snake_case_		,  additional_replacements=[meta_path]						)

       lowercase__:							List[Any]         =     renew_temp_conv_paths(snake_case_						)
       lowercase__:							Union[str, Any]         =     {"old": "middle_block.3.temopral_conv", "new": "mid_block.temp_convs.1"}
       assign_to_checkpoint(
           snake_case_		,  snake_case_		,  snake_case_		,  config=snake_case_		,  additional_replacements=[meta_path]						)

       lowercase__:							Tuple         =     renew_attention_paths(snake_case_						)
       lowercase__:							str         =     {"old": "middle_block.1", "new": "mid_block.attentions.0"}
       assign_to_checkpoint(
           snake_case_		,  snake_case_		,  snake_case_		,  additional_replacements=[meta_path]		,  config=snake_case_						)

       lowercase__:							Any         =     renew_attention_paths(snake_case_						)
       lowercase__:							Any         =     {"old": "middle_block.2", "new": "mid_block.temp_attentions.0"}
       assign_to_checkpoint(
           snake_case_		,  snake_case_		,  snake_case_		,  additional_replacements=[meta_path]		,  config=snake_case_						)

       for i in range(snake_case_						):
              lowercase__:							int         =     i // (config["layers_per_block"] + 1)
              lowercase__:							Tuple         =     i % (config["layers_per_block"] + 1)
              lowercase__:							Optional[int]         =     [shave_segments(snake_case_		,  2						) for name in output_blocks[i]]
              lowercase__:							Any         =     {}

              for layer in output_block_layers:
                     lowercase__    ,      lowercase__:							str         =     layer.split("."						)[0], shave_segments(snake_case_		,  1						)
                     if layer_id in output_block_list:
                            output_block_list[layer_id].append(snake_case_						)
                     else:
                            lowercase__:							Union[str, Any]         =     [layer_name]

              if len(snake_case_						) > 1:
                     lowercase__:							Tuple         =     [key for key in output_blocks[i] if F"""output_blocks.{i}.0""" in key]
                     lowercase__:							Dict         =     [key for key in output_blocks[i] if F"""output_blocks.{i}.1""" in key]
                     lowercase__:							int         =     [key for key in output_blocks[i] if F"""output_blocks.{i}.2""" in key]

                     lowercase__:							Tuple         =     renew_resnet_paths(snake_case_						)
                     lowercase__:							int         =     renew_resnet_paths(snake_case_						)

                     lowercase__:							List[Any]         =     {"old": F"""output_blocks.{i}.0""", "new": F"""up_blocks.{block_id}.resnets.{layer_in_block_id}"""}
                     assign_to_checkpoint(
                         snake_case_		,  snake_case_		,  snake_case_		,  additional_replacements=[meta_path]		,  config=snake_case_						)

                     lowercase__:							Dict         =     [key for key in resnets if "temopral_conv" in key]
                     lowercase__:							Dict         =     renew_temp_conv_paths(snake_case_						)
                     lowercase__:							Any         =     {
                         "old": F"""output_blocks.{i}.0.temopral_conv""",
                         "new": F"""up_blocks.{block_id}.temp_convs.{layer_in_block_id}""",
                     }
                     assign_to_checkpoint(
                         snake_case_		,  snake_case_		,  snake_case_		,  additional_replacements=[meta_path]		,  config=snake_case_						)

                     lowercase__:							Any         =     {k: sorted(snake_case_						) for k, v in output_block_list.items()}
                     if ["conv.bias", "conv.weight"] in output_block_list.values():
                            lowercase__:							Dict         =     list(output_block_list.values()						).index(["conv.bias", "conv.weight"]						)
                            lowercase__:							Optional[int]         =     unet_state_dict[
                                F"""output_blocks.{i}.{index}.conv.weight"""
                            ]
                            lowercase__:							Optional[Any]         =     unet_state_dict[
                                F"""output_blocks.{i}.{index}.conv.bias"""
                            ]

                            # Clear attentions as they have been attributed above.
                            if len(snake_case_						) == 2:
                                   lowercase__:							Dict         =     []

                     if len(snake_case_						):
                            lowercase__:							List[Any]         =     renew_attention_paths(snake_case_						)
                            lowercase__:							Dict         =     {
                                "old": F"""output_blocks.{i}.1""",
                                "new": F"""up_blocks.{block_id}.attentions.{layer_in_block_id}""",
                            }
                            assign_to_checkpoint(
                                snake_case_		,  snake_case_		,  snake_case_		,  additional_replacements=[meta_path]		,  config=snake_case_						)

                     if len(snake_case_						):
                            lowercase__:							Dict         =     renew_attention_paths(snake_case_						)
                            lowercase__:							Union[str, Any]         =     {
                                "old": F"""output_blocks.{i}.2""",
                                "new": F"""up_blocks.{block_id}.temp_attentions.{layer_in_block_id}""",
                            }
                            assign_to_checkpoint(
                                snake_case_		,  snake_case_		,  snake_case_		,  additional_replacements=[meta_path]		,  config=snake_case_						)
              else:
                     lowercase__:							Dict         =     renew_resnet_paths(snake_case_		,  n_shave_prefix_segments=1						)
                     for path in resnet_0_paths:
                            lowercase__:							Union[str, Any]         =     ".".join(["output_blocks", str(snake_case_						), path["old"]]						)
                            lowercase__:							Tuple         =     ".".join(["up_blocks", str(snake_case_						), "resnets", str(snake_case_						), path["new"]]						)
                            lowercase__:							Tuple         =     unet_state_dict[old_path]

                     lowercase__:							Optional[Any]         =     [l for l in output_block_layers if "temopral_conv" in l]
                     for path in temopral_conv_paths:
                            lowercase__:							Tuple         =     path.split("temopral_conv."						)[-1]
                            lowercase__:							List[Any]         =     ".".join(["output_blocks", str(snake_case_						), str(snake_case_						), "temopral_conv", pruned_path]						)
                            lowercase__:							int         =     ".".join(["up_blocks", str(snake_case_						), "temp_convs", str(snake_case_						), pruned_path]						)
                            lowercase__:							int         =     unet_state_dict[old_path]

       return new_checkpoint


if __name__ == "__main__":
  UpperCamelCase					=     argparse.ArgumentParser()

  parser.add_argument(
      """--checkpoint_path""", default=None, type=str, required=True, help="""Path to the checkpoint to convert."""
  )
  parser.add_argument("""--dump_path""", default=None, type=str, required=True, help="""Path to the output model.""")
  UpperCamelCase					=     parser.parse_args()

  UpperCamelCase					=     torch.load(args.checkpoint_path, map_location="""cpu""")
  UpperCamelCase					=     UNetaDConditionModel()

  UpperCamelCase					=     convert_ldm_unet_checkpoint(unet_checkpoint, unet.config)

  UpperCamelCase					=     set(unet.state_dict().keys()) - set(converted_ckpt.keys())
  UpperCamelCase					=     set(converted_ckpt.keys()) - set(unet.state_dict().keys())

  assert len(diff_a) == len(diff_a) == 0, "Converted weights don't match"

  # load state_dict
  unet.load_state_dict(converted_ckpt)

  unet.save_pretrained(args.dump_path)

  # -- finish converting the unet --








import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    MODEL_FOR_PRETRAINING_MAPPING,
				    FunnelBaseModel,
				    FunnelForMaskedLM,
				    FunnelForMultipleChoice,
				    FunnelForPreTraining,
				    FunnelForQuestionAnswering,
				    FunnelForSequenceClassification,
				    FunnelForTokenClassification,
				    FunnelModel,
				)






class 							SCREAMING_SNAKE_CASE__     :

							"""simple docstring"""






							def __init__(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_=13  ,						SCREAMING_SNAKE_CASE_=7  ,						SCREAMING_SNAKE_CASE_=True  ,						SCREAMING_SNAKE_CASE_=True  ,						SCREAMING_SNAKE_CASE_=True  ,						SCREAMING_SNAKE_CASE_=True  ,						SCREAMING_SNAKE_CASE_=99  ,						SCREAMING_SNAKE_CASE_=[1, 1, 2]  ,						SCREAMING_SNAKE_CASE_=1  ,						SCREAMING_SNAKE_CASE_=32  ,						SCREAMING_SNAKE_CASE_=4  ,						SCREAMING_SNAKE_CASE_=8  ,						SCREAMING_SNAKE_CASE_=37  ,						SCREAMING_SNAKE_CASE_="gelu_new"  ,						SCREAMING_SNAKE_CASE_=0.1  ,						SCREAMING_SNAKE_CASE_=0.1  ,						SCREAMING_SNAKE_CASE_=0.0  ,						SCREAMING_SNAKE_CASE_=512  ,						SCREAMING_SNAKE_CASE_=3  ,						SCREAMING_SNAKE_CASE_=0.0_2  ,						SCREAMING_SNAKE_CASE_=3  ,						SCREAMING_SNAKE_CASE_=4  ,						SCREAMING_SNAKE_CASE_=None  ,						SCREAMING_SNAKE_CASE_=False  ,						)-> str:





													'''simple docstring'''






													__UpperCamelCase  	=      parent
													__UpperCamelCase  	=      batch_size
													__UpperCamelCase  	=      seq_length
													__UpperCamelCase  	=      is_training
													__UpperCamelCase  	=      use_input_mask
													__UpperCamelCase  	=      use_token_type_ids
													__UpperCamelCase  	=      use_labels
													__UpperCamelCase  	=      vocab_size
													__UpperCamelCase  	=      block_sizes
													__UpperCamelCase  	=      num_decoder_layers
													__UpperCamelCase  	=      d_model
													__UpperCamelCase  	=      n_head
													__UpperCamelCase  	=      d_head
													__UpperCamelCase  	=      d_inner
													__UpperCamelCase  	=      hidden_act
													__UpperCamelCase  	=      hidden_dropout
													__UpperCamelCase  	=      attention_dropout
													__UpperCamelCase  	=      activation_dropout
													__UpperCamelCase  	=      max_position_embeddings
													__UpperCamelCase  	=      type_vocab_size
													__UpperCamelCase  	=      2
													__UpperCamelCase  	=      num_labels
													__UpperCamelCase  	=      num_choices
													__UpperCamelCase  	=      scope
													__UpperCamelCase  	=      initializer_std

													# Used in the tests to check the size of the first attention layer
													__UpperCamelCase  	=      n_head
													# Used in the tests to check the size of the first hidden state
													__UpperCamelCase  	=      self.d_model
													# Used in the tests to check the number of output hidden states/attentions
													__UpperCamelCase  	=      sum(self.block_sizes      ) + (0 if base else self.num_decoder_layers)
													# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
													# the last hidden state of the first block (which is the first hidden state of the decoder).
													if not base:
																			__UpperCamelCase  	=      self.num_hidden_layers + 2



							def      A__		(	self      )-> Any:





													'''simple docstring'''






													__UpperCamelCase  	=      ids_tensor([self.batch_size, self.seq_length]  ,						self.vocab_size      )

													__UpperCamelCase  	=      None
													if self.use_input_mask:
																			__UpperCamelCase  	=      ids_tensor([self.batch_size, self.seq_length]  ,						vocab_size=2      )

													__UpperCamelCase  	=      None
													if self.use_token_type_ids:
																			__UpperCamelCase  	=      ids_tensor([self.batch_size, self.seq_length]  ,						self.type_vocab_size      )

													__UpperCamelCase  	=      None
													__UpperCamelCase  	=      None
													__UpperCamelCase  	=      None
													if self.use_labels:
																			__UpperCamelCase  	=      ids_tensor([self.batch_size]  ,						self.type_sequence_label_size      )
																			__UpperCamelCase  	=      ids_tensor([self.batch_size, self.seq_length]  ,						self.num_labels      )
																			__UpperCamelCase  	=      ids_tensor([self.batch_size]  ,						self.num_choices      )
																			__UpperCamelCase  	=      ids_tensor([self.batch_size, self.seq_length]  ,						1      )

													__UpperCamelCase  	=      self.get_config()

													return (
													    config,
													    input_ids,
													    token_type_ids,
													    input_mask,
													    sequence_labels,
													    token_labels,
													    choice_labels,
													    fake_token_labels,
													)



							def      A__		(	self      )-> List[str]:





													'''simple docstring'''






													return FunnelConfig(
													    vocab_size=self.vocab_size  ,						block_sizes=self.block_sizes  ,						num_decoder_layers=self.num_decoder_layers  ,						d_model=self.d_model  ,						n_head=self.n_head  ,						d_head=self.d_head  ,						d_inner=self.d_inner  ,						hidden_act=self.hidden_act  ,						hidden_dropout=self.hidden_dropout  ,						attention_dropout=self.attention_dropout  ,						activation_dropout=self.activation_dropout  ,						max_position_embeddings=self.max_position_embeddings  ,						type_vocab_size=self.type_vocab_size  ,						initializer_std=self.initializer_std  ,						)



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						)-> str:





													'''simple docstring'''






													__UpperCamelCase  	=      FunnelModel(config=SCREAMING_SNAKE_CASE_      )
													model.to(SCREAMING_SNAKE_CASE_      )
													model.eval()
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_  ,						attention_mask=SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_      )
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_      )
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_      )
													self.parent.assertEqual(result.last_hidden_state.shape  ,						(self.batch_size, self.seq_length, self.d_model)      )

													__UpperCamelCase  	=      False
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_      )
													self.parent.assertEqual(result.last_hidden_state.shape  ,						(self.batch_size, self.seq_length, self.d_model)      )

													__UpperCamelCase  	=      False
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_      )
													self.parent.assertEqual(result.last_hidden_state.shape  ,						(self.batch_size, self.seq_length, self.d_model)      )



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						)-> Optional[Any]:





													'''simple docstring'''






													__UpperCamelCase  	=      FunnelBaseModel(config=SCREAMING_SNAKE_CASE_      )
													model.to(SCREAMING_SNAKE_CASE_      )
													model.eval()
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_  ,						attention_mask=SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_      )
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_      )
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_      )
													self.parent.assertEqual(result.last_hidden_state.shape  ,						(self.batch_size, 2, self.d_model)      )

													__UpperCamelCase  	=      False
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_      )
													self.parent.assertEqual(result.last_hidden_state.shape  ,						(self.batch_size, 3, self.d_model)      )

													__UpperCamelCase  	=      False
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_      )
													self.parent.assertEqual(result.last_hidden_state.shape  ,						(self.batch_size, 2, self.d_model)      )



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						)-> List[Any]:





													'''simple docstring'''






													__UpperCamelCase  	=      self.num_labels
													__UpperCamelCase  	=      FunnelForPreTraining(config=SCREAMING_SNAKE_CASE_      )
													model.to(SCREAMING_SNAKE_CASE_      )
													model.eval()
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_  ,						attention_mask=SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_  ,						labels=SCREAMING_SNAKE_CASE_      )
													self.parent.assertEqual(result.logits.shape  ,						(self.batch_size, self.seq_length)      )



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						)-> Dict:





													'''simple docstring'''






													__UpperCamelCase  	=      FunnelForMaskedLM(config=SCREAMING_SNAKE_CASE_      )
													model.to(SCREAMING_SNAKE_CASE_      )
													model.eval()
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_  ,						attention_mask=SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_  ,						labels=SCREAMING_SNAKE_CASE_      )
													self.parent.assertEqual(result.logits.shape  ,						(self.batch_size, self.seq_length, self.vocab_size)      )



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						)-> List[Any]:





													'''simple docstring'''






													__UpperCamelCase  	=      self.num_labels
													__UpperCamelCase  	=      FunnelForSequenceClassification(SCREAMING_SNAKE_CASE_      )
													model.to(SCREAMING_SNAKE_CASE_      )
													model.eval()
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_  ,						attention_mask=SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_  ,						labels=SCREAMING_SNAKE_CASE_      )
													self.parent.assertEqual(result.logits.shape  ,						(self.batch_size, self.num_labels)      )



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						)-> Dict:





													'''simple docstring'''






													__UpperCamelCase  	=      self.num_choices
													__UpperCamelCase  	=      FunnelForMultipleChoice(config=SCREAMING_SNAKE_CASE_      )
													model.to(SCREAMING_SNAKE_CASE_      )
													model.eval()
													__UpperCamelCase  	=      input_ids.unsqueeze(1      ).expand(-1  ,						self.num_choices  ,						-1      ).contiguous()
													__UpperCamelCase  	=      token_type_ids.unsqueeze(1      ).expand(-1  ,						self.num_choices  ,						-1      ).contiguous()
													__UpperCamelCase  	=      input_mask.unsqueeze(1      ).expand(-1  ,						self.num_choices  ,						-1      ).contiguous()
													__UpperCamelCase  	=      model(
													    SCREAMING_SNAKE_CASE_  ,						attention_mask=SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_  ,						labels=SCREAMING_SNAKE_CASE_  ,						)
													self.parent.assertEqual(result.logits.shape  ,						(self.batch_size, self.num_choices)      )



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						)-> Optional[Any]:





													'''simple docstring'''






													__UpperCamelCase  	=      self.num_labels
													__UpperCamelCase  	=      FunnelForTokenClassification(config=SCREAMING_SNAKE_CASE_      )
													model.to(SCREAMING_SNAKE_CASE_      )
													model.eval()
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_  ,						attention_mask=SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_  ,						labels=SCREAMING_SNAKE_CASE_      )
													self.parent.assertEqual(result.logits.shape  ,						(self.batch_size, self.seq_length, self.num_labels)      )



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						)-> Tuple:





													'''simple docstring'''






													__UpperCamelCase  	=      FunnelForQuestionAnswering(config=SCREAMING_SNAKE_CASE_      )
													model.to(SCREAMING_SNAKE_CASE_      )
													model.eval()
													__UpperCamelCase  	=      model(
													    SCREAMING_SNAKE_CASE_  ,						attention_mask=SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_  ,						start_positions=SCREAMING_SNAKE_CASE_  ,						end_positions=SCREAMING_SNAKE_CASE_  ,						)
													self.parent.assertEqual(result.start_logits.shape  ,						(self.batch_size, self.seq_length)      )
													self.parent.assertEqual(result.end_logits.shape  ,						(self.batch_size, self.seq_length)      )



							def      A__		(	self      )-> Dict:





													'''simple docstring'''






													__UpperCamelCase  	=      self.prepare_config_and_inputs()
													(
													    (
													    __UpperCamelCase
													)	,				(
													    __UpperCamelCase
													)	,				(
													    __UpperCamelCase
													)	,				(
													    __UpperCamelCase
													)	,				(
													    __UpperCamelCase
													)	,				(
													    __UpperCamelCase
													)	,				(
													    __UpperCamelCase
													)	,				(
													    __UpperCamelCase
													)	,				
													)  	=      config_and_inputs
													__UpperCamelCase  	=      {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
													return config, inputs_dict






@require_torch
class 							SCREAMING_SNAKE_CASE__     (				SCREAMING_SNAKE_CASE_ ,		SCREAMING_SNAKE_CASE_ ,		unittest.TestCase     ):

							"""simple docstring"""






							_snake_case  		=	False
							_snake_case  		=	False
							_snake_case  		=	(
							    (
							        FunnelModel,
							        FunnelForMaskedLM,
							        FunnelForPreTraining,
							        FunnelForQuestionAnswering,
							        FunnelForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							_snake_case  		=	(
							    {
							        'feature-extraction': (FunnelBaseModel, FunnelModel),
							        'fill-mask': FunnelForMaskedLM,
							        'question-answering': FunnelForQuestionAnswering,
							        'text-classification': FunnelForSequenceClassification,
							        'token-classification': FunnelForTokenClassification,
							        'zero-shot': FunnelForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_=False      )-> str:





													'''simple docstring'''






													__UpperCamelCase  	=      super()._prepare_for_class(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						return_labels=SCREAMING_SNAKE_CASE_      )

													if return_labels:
																			if model_class in get_values(SCREAMING_SNAKE_CASE_      ):
																									__UpperCamelCase  	=      torch.zeros(
																									    (self.model_tester.batch_size, self.model_tester.seq_length)  ,						dtype=torch.long  ,						device=SCREAMING_SNAKE_CASE_      )
													return inputs_dict



							def      A__		(	self      )-> Tuple:





													'''simple docstring'''






													__UpperCamelCase  	=      FunnelModelTester(self      )
													__UpperCamelCase  	=      ConfigTester(self  ,						config_class=SCREAMING_SNAKE_CASE_      )



							def      A__		(	self      )-> str:





													'''simple docstring'''






													self.config_tester.run_common_tests()



							def      A__		(	self      )-> Dict:





													'''simple docstring'''






													__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_      )



							def      A__		(	self      )-> Dict:





													'''simple docstring'''






													__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_pretraining(*SCREAMING_SNAKE_CASE_      )



							def      A__		(	self      )-> Optional[Any]:





													'''simple docstring'''






													__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_masked_lm(*SCREAMING_SNAKE_CASE_      )



							def      A__		(	self      )-> Optional[int]:





													'''simple docstring'''






													__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_token_classification(*SCREAMING_SNAKE_CASE_      )



							def      A__		(	self      )-> int:





													'''simple docstring'''






													__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_question_answering(*SCREAMING_SNAKE_CASE_      )



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_      )-> Optional[int]:





													'''simple docstring'''






													if hasattr(SCREAMING_SNAKE_CASE_  ,						'''weight'''      ) and module.weight is not None:
																			module.weight.data.fill_(3      )
													if hasattr(SCREAMING_SNAKE_CASE_  ,						'''bias'''      ) and module.bias is not None:
																			module.bias.data.fill_(3      )

													for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
																			if hasattr(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_      ) and getattr(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_      ) is not None:
																									__UpperCamelCase  	=      getattr(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_      )
																									weight.data.fill_(3      )






@require_torch
class 							SCREAMING_SNAKE_CASE__     (				SCREAMING_SNAKE_CASE_ ,		unittest.TestCase     ):

							"""simple docstring"""






							_snake_case  		=	False
							_snake_case  		=	False
							_snake_case  		=	(
							    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
							)



							def      A__		(	self      )-> Optional[Any]:





													'''simple docstring'''






													__UpperCamelCase  	=      FunnelModelTester(self  ,						base=SCREAMING_SNAKE_CASE_      )
													__UpperCamelCase  	=      ConfigTester(self  ,						config_class=SCREAMING_SNAKE_CASE_      )



							def      A__		(	self      )-> List[str]:





													'''simple docstring'''






													self.config_tester.run_common_tests()



							def      A__		(	self      )-> Tuple:





													'''simple docstring'''






													__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_base_model(*SCREAMING_SNAKE_CASE_      )



							def      A__		(	self      )-> Dict:





													'''simple docstring'''






													__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_sequence_classification(*SCREAMING_SNAKE_CASE_      )



							def      A__		(	self      )-> Optional[Any]:





													'''simple docstring'''






													__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_multiple_choice(*SCREAMING_SNAKE_CASE_      )



							def      A__		(	self      )-> List[Any]:





													'''simple docstring'''






													__UpperCamelCase	,				__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs_for_common()
													__UpperCamelCase  	=      True

													for model_class in self.all_model_classes:
																			if model_class.__name__ == "FunnelBaseModel":
																									continue
																			__UpperCamelCase  	=      model_class(SCREAMING_SNAKE_CASE_      )
																			model.to(SCREAMING_SNAKE_CASE_      )
																			model.train()
																			__UpperCamelCase  	=      self._prepare_for_class(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						return_labels=SCREAMING_SNAKE_CASE_      )
																			__UpperCamelCase  	=      model(**SCREAMING_SNAKE_CASE_      ).loss
																			loss.backward()



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_      )-> str:





													'''simple docstring'''






													if hasattr(SCREAMING_SNAKE_CASE_  ,						'''weight'''      ) and module.weight is not None:
																			module.weight.data.fill_(3      )
													if hasattr(SCREAMING_SNAKE_CASE_  ,						'''bias'''      ) and module.bias is not None:
																			module.bias.data.fill_(3      )

													for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
																			if hasattr(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_      ) and getattr(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_      ) is not None:
																									__UpperCamelCase  	=      getattr(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_      )
																									weight.data.fill_(3      )






@require_torch
@require_sentencepiece
@require_tokenizers
class 							SCREAMING_SNAKE_CASE__     (				unittest.TestCase     ):

							"""simple docstring"""






							def      A__		(	self      )-> Optional[Any]:





													'''simple docstring'''






													__UpperCamelCase  	=      13
													__UpperCamelCase  	=      7
													__UpperCamelCase  	=      torch.arange(0  ,						batch_size * sequence_length      ).long().reshape(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_      )
													__UpperCamelCase  	=      [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
													__UpperCamelCase  	=      torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]      )

													__UpperCamelCase  	=      FunnelModel.from_pretrained('''sgugger/funnel-random-tiny'''      )
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_      )[0].abs()

													__UpperCamelCase  	=      torch.tensor(2_3_4_4.8_3_5_2      )
													__UpperCamelCase  	=      torch.tensor(0.8_0_5_2      )
													self.assertTrue(torch.allclose(output.sum()  ,						SCREAMING_SNAKE_CASE_  ,						atol=1E-4      )      )
													self.assertTrue(torch.allclose(output.mean()  ,						SCREAMING_SNAKE_CASE_  ,						atol=1E-4      )      )

													__UpperCamelCase  	=      torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]      )
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_  ,						attention_mask=SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_      )[0].abs()

													__UpperCamelCase  	=      torch.tensor(2_3_4_3.8_4_2_5      )
													__UpperCamelCase  	=      torch.tensor(0.8_0_4_9      )
													self.assertTrue(torch.allclose(output.sum()  ,						SCREAMING_SNAKE_CASE_  ,						atol=1E-4      )      )
													self.assertTrue(torch.allclose(output.mean()  ,						SCREAMING_SNAKE_CASE_  ,						atol=1E-4      )      )



							@slow
							def      A__		(	self      )-> Union[str, Any]:





													'''simple docstring'''






													__UpperCamelCase  	=      FunnelTokenizer.from_pretrained('''huggingface/funnel-small'''      )
													__UpperCamelCase  	=      FunnelModel.from_pretrained('''huggingface/funnel-small'''      )
													__UpperCamelCase  	=      tokenizer('''Hello! I am the Funnel Transformer model.'''  ,						return_tensors='''pt'''      )
													__UpperCamelCase  	=      model(**SCREAMING_SNAKE_CASE_      )[0]

													__UpperCamelCase  	=      torch.tensor(2_3_5.7_2_4_6      )
													__UpperCamelCase  	=      torch.tensor(0.0_2_5_6      )
													self.assertTrue(torch.allclose(output.sum()  ,						SCREAMING_SNAKE_CASE_  ,						atol=1E-4      )      )
													self.assertTrue(torch.allclose(output.mean()  ,						SCREAMING_SNAKE_CASE_  ,						atol=1E-4      )      )




