



'''simple docstring'''






import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import cached_download, hf_hub_download, hf_hub_url
from PIL import Image

from transformers import DetaConfig, DetaForObjectDetection, DetaImageProcessor
from transformers.utils import logging


logging.set_verbosity_info()
lowerCamelCase__	  =		logging.get_logger(__name__)





def 				_SCREAMING_SNAKE_CASE(					)		->Dict:






							'''simple docstring'''
							_lowercase							:		Union[str, Any]				=   DetaConfig(
							    num_queries=9_00 ,       encoder_ffn_dim=20_48 ,       decoder_ffn_dim=20_48 ,       num_feature_levels=5 ,       assign_first_stage=snake_case_ ,       with_box_refine=snake_case_ ,       two_stage=snake_case_ ,       )

							# set labels
							_lowercase							:		Optional[Any]				=   91
							_lowercase							:		List[Any]				=   '''huggingface/label-files'''
							_lowercase							:		Optional[int]				=   '''coco-detection-id2label.json'''
							_lowercase							:		Any				=   json.load(open(cached_download(hf_hub_url(snake_case_ ,       snake_case_ ,       repo_type='''dataset'''  )  ) ,       '''r'''  )  )
							_lowercase							:		str				=   {int(snake_case_  ): v for k, v in idalabel.items()}
							_lowercase							:		Dict				=   idalabel
							_lowercase							:		Tuple				=   {v: k for k, v in idalabel.items()}

							return config





def 				_SCREAMING_SNAKE_CASE(					snake_case_    :     List[str]  )		->Union[str, Any]:






							'''simple docstring'''
							_lowercase							:		str				=   []

							# stem
							# fmt: off
							rename_keys.append(('''backbone.0.body.conv1.weight''', '''model.backbone.model.embedder.embedder.convolution.weight''')  )
							rename_keys.append(('''backbone.0.body.bn1.weight''', '''model.backbone.model.embedder.embedder.normalization.weight''')  )
							rename_keys.append(('''backbone.0.body.bn1.bias''', '''model.backbone.model.embedder.embedder.normalization.bias''')  )
							rename_keys.append(('''backbone.0.body.bn1.running_mean''', '''model.backbone.model.embedder.embedder.normalization.running_mean''')  )
							rename_keys.append(('''backbone.0.body.bn1.running_var''', '''model.backbone.model.embedder.embedder.normalization.running_var''')  )
							# stages
							for stage_idx in range(len(config.backbone_config.depths  )  ):
														for layer_idx in range(config.backbone_config.depths[stage_idx]  ):
																					# shortcut
																					if layer_idx == 0:
																												rename_keys.append(
																												    (
																												        F"backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.0.weight",
																												        F"model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.convolution.weight",
																												    )  )
																												rename_keys.append(
																												    (
																												        F"backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.weight",
																												        F"model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.weight",
																												    )  )
																												rename_keys.append(
																												    (
																												        F"backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.bias",
																												        F"model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.bias",
																												    )  )
																												rename_keys.append(
																												    (
																												        F"backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.running_mean",
																												        F"model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.running_mean",
																												    )  )
																												rename_keys.append(
																												    (
																												        F"backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.running_var",
																												        F"model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.running_var",
																												    )  )
																					# 3 convs
																					for i in range(3  ):
																												rename_keys.append(
																												    (
																												        F"backbone.0.body.layer{stage_idx + 1}.{layer_idx}.conv{i+1}.weight",
																												        F"model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.convolution.weight",
																												    )  )
																												rename_keys.append(
																												    (
																												        F"backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.weight",
																												        F"model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.weight",
																												    )  )
																												rename_keys.append(
																												    (
																												        F"backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.bias",
																												        F"model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.bias",
																												    )  )
																												rename_keys.append(
																												    (
																												        F"backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.running_mean",
																												        F"model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.running_mean",
																												    )  )
																												rename_keys.append(
																												    (
																												        F"backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.running_var",
																												        F"model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.running_var",
																												    )  )
    # transformer encoder
							for i in range(config.encoder_layers  ):
														rename_keys.append((F"transformer.encoder.layers.{i}.self_attn.sampling_offsets.weight", F"model.encoder.layers.{i}.self_attn.sampling_offsets.weight")  )
														rename_keys.append((F"transformer.encoder.layers.{i}.self_attn.sampling_offsets.bias", F"model.encoder.layers.{i}.self_attn.sampling_offsets.bias")  )
														rename_keys.append((F"transformer.encoder.layers.{i}.self_attn.attention_weights.weight", F"model.encoder.layers.{i}.self_attn.attention_weights.weight")  )
														rename_keys.append((F"transformer.encoder.layers.{i}.self_attn.attention_weights.bias", F"model.encoder.layers.{i}.self_attn.attention_weights.bias")  )
														rename_keys.append((F"transformer.encoder.layers.{i}.self_attn.value_proj.weight", F"model.encoder.layers.{i}.self_attn.value_proj.weight")  )
														rename_keys.append((F"transformer.encoder.layers.{i}.self_attn.value_proj.bias", F"model.encoder.layers.{i}.self_attn.value_proj.bias")  )
														rename_keys.append((F"transformer.encoder.layers.{i}.self_attn.output_proj.weight", F"model.encoder.layers.{i}.self_attn.output_proj.weight")  )
														rename_keys.append((F"transformer.encoder.layers.{i}.self_attn.output_proj.bias", F"model.encoder.layers.{i}.self_attn.output_proj.bias")  )
														rename_keys.append((F"transformer.encoder.layers.{i}.norm1.weight", F"model.encoder.layers.{i}.self_attn_layer_norm.weight")  )
														rename_keys.append((F"transformer.encoder.layers.{i}.norm1.bias", F"model.encoder.layers.{i}.self_attn_layer_norm.bias")  )
														rename_keys.append((F"transformer.encoder.layers.{i}.linear1.weight", F"model.encoder.layers.{i}.fc1.weight")  )
														rename_keys.append((F"transformer.encoder.layers.{i}.linear1.bias", F"model.encoder.layers.{i}.fc1.bias")  )
														rename_keys.append((F"transformer.encoder.layers.{i}.linear2.weight", F"model.encoder.layers.{i}.fc2.weight")  )
														rename_keys.append((F"transformer.encoder.layers.{i}.linear2.bias", F"model.encoder.layers.{i}.fc2.bias")  )
														rename_keys.append((F"transformer.encoder.layers.{i}.norm2.weight", F"model.encoder.layers.{i}.final_layer_norm.weight")  )
														rename_keys.append((F"transformer.encoder.layers.{i}.norm2.bias", F"model.encoder.layers.{i}.final_layer_norm.bias")  )

							# transformer decoder
							for i in range(config.decoder_layers  ):
														rename_keys.append((F"transformer.decoder.layers.{i}.cross_attn.sampling_offsets.weight", F"model.decoder.layers.{i}.encoder_attn.sampling_offsets.weight")  )
														rename_keys.append((F"transformer.decoder.layers.{i}.cross_attn.sampling_offsets.bias", F"model.decoder.layers.{i}.encoder_attn.sampling_offsets.bias")  )
														rename_keys.append((F"transformer.decoder.layers.{i}.cross_attn.attention_weights.weight", F"model.decoder.layers.{i}.encoder_attn.attention_weights.weight")  )
														rename_keys.append((F"transformer.decoder.layers.{i}.cross_attn.attention_weights.bias", F"model.decoder.layers.{i}.encoder_attn.attention_weights.bias")  )
														rename_keys.append((F"transformer.decoder.layers.{i}.cross_attn.value_proj.weight", F"model.decoder.layers.{i}.encoder_attn.value_proj.weight")  )
														rename_keys.append((F"transformer.decoder.layers.{i}.cross_attn.value_proj.bias", F"model.decoder.layers.{i}.encoder_attn.value_proj.bias")  )
														rename_keys.append((F"transformer.decoder.layers.{i}.cross_attn.output_proj.weight", F"model.decoder.layers.{i}.encoder_attn.output_proj.weight")  )
														rename_keys.append((F"transformer.decoder.layers.{i}.cross_attn.output_proj.bias", F"model.decoder.layers.{i}.encoder_attn.output_proj.bias")  )
														rename_keys.append((F"transformer.decoder.layers.{i}.norm1.weight", F"model.decoder.layers.{i}.encoder_attn_layer_norm.weight")  )
														rename_keys.append((F"transformer.decoder.layers.{i}.norm1.bias", F"model.decoder.layers.{i}.encoder_attn_layer_norm.bias")  )
														rename_keys.append((F"transformer.decoder.layers.{i}.self_attn.out_proj.weight", F"model.decoder.layers.{i}.self_attn.out_proj.weight")  )
														rename_keys.append((F"transformer.decoder.layers.{i}.self_attn.out_proj.bias", F"model.decoder.layers.{i}.self_attn.out_proj.bias")  )
														rename_keys.append((F"transformer.decoder.layers.{i}.norm2.weight", F"model.decoder.layers.{i}.self_attn_layer_norm.weight")  )
														rename_keys.append((F"transformer.decoder.layers.{i}.norm2.bias", F"model.decoder.layers.{i}.self_attn_layer_norm.bias")  )
														rename_keys.append((F"transformer.decoder.layers.{i}.linear1.weight", F"model.decoder.layers.{i}.fc1.weight")  )
														rename_keys.append((F"transformer.decoder.layers.{i}.linear1.bias", F"model.decoder.layers.{i}.fc1.bias")  )
														rename_keys.append((F"transformer.decoder.layers.{i}.linear2.weight", F"model.decoder.layers.{i}.fc2.weight")  )
														rename_keys.append((F"transformer.decoder.layers.{i}.linear2.bias", F"model.decoder.layers.{i}.fc2.bias")  )
														rename_keys.append((F"transformer.decoder.layers.{i}.norm3.weight", F"model.decoder.layers.{i}.final_layer_norm.weight")  )
														rename_keys.append((F"transformer.decoder.layers.{i}.norm3.bias", F"model.decoder.layers.{i}.final_layer_norm.bias")  )

							# fmt: on

							return rename_keys





def 				_SCREAMING_SNAKE_CASE(					snake_case_    :     List[Any] ,       snake_case_    :     Optional[int] ,       snake_case_    :     Any  )		->Union[str, Any]:






							'''simple docstring'''
							_lowercase							:		Tuple				=   dct.pop(snake_case_  )
							_lowercase							:		Tuple				=   val





def 				_SCREAMING_SNAKE_CASE(					snake_case_    :     Optional[int] ,       snake_case_    :     List[Any]  )		->Tuple:






							'''simple docstring'''
							# transformer decoder self-attention layers
							_lowercase							:		Dict				=   config.d_model
							for i in range(config.decoder_layers  ):
														# read in weights + bias of input projection layer of self-attention
														_lowercase							:		List[str]				=   state_dict.pop(F"transformer.decoder.layers.{i}.self_attn.in_proj_weight"  )
														_lowercase							:		List[str]				=   state_dict.pop(F"transformer.decoder.layers.{i}.self_attn.in_proj_bias"  )
														# next, add query, keys and values (in that order) to the state dict
														_lowercase							:		Union[str, Any]				=   in_proj_weight[:hidden_size, :]
														_lowercase							:		str				=   in_proj_bias[:hidden_size]
														_lowercase							:		List[str]				=   in_proj_weight[
														    hidden_size : hidden_size * 2, :
														]
														_lowercase							:		Union[str, Any]				=   in_proj_bias[hidden_size : hidden_size * 2]
														_lowercase							:		List[Any]				=   in_proj_weight[-hidden_size:, :]
														_lowercase							:		Tuple				=   in_proj_bias[-hidden_size:]





def 				_SCREAMING_SNAKE_CASE(					)		->Dict:






							'''simple docstring'''
							_lowercase							:		Optional[Any]				=   '''http://images.cocodataset.org/val2017/000000039769.jpg'''
							_lowercase							:		List[str]				=   Image.open(requests.get(snake_case_ ,       stream=snake_case_  ).raw  )

							return im





@torch.no_grad()
def 				_SCREAMING_SNAKE_CASE(					snake_case_    :     Tuple ,       snake_case_    :     List[str] ,       snake_case_    :     Any  )		->Union[str, Any]:






							'''simple docstring'''
							_lowercase							:		Any				=   get_deta_config()

							# load original state dict
							if model_name == "deta-resnet-50":
														_lowercase							:		List[str]				=   '''adet_checkpoint0011.pth'''
							elif model_name == "deta-resnet-50-24-epochs":
														_lowercase							:		Optional[Any]				=   '''adet_2x_checkpoint0023.pth'''
							else:
														raise ValueError(F"Model name {model_name} not supported"  )
							_lowercase							:		Dict				=   hf_hub_download(repo_id='''nielsr/deta-checkpoints''' ,       filename=snake_case_  )
							_lowercase							:		Tuple				=   torch.load(snake_case_ ,       map_location='''cpu'''  )['''model''']

							# rename keys
							_lowercase							:		Optional[int]				=   create_rename_keys(snake_case_  )
							for src, dest in rename_keys:
														rename_key(snake_case_ ,       snake_case_ ,       snake_case_  )
							read_in_decoder_q_k_v(snake_case_ ,       snake_case_  )

							# fix some prefixes
							for key in state_dict.copy().keys():
														if "transformer.decoder.class_embed" in key or "transformer.decoder.bbox_embed" in key:
																					_lowercase							:		Union[str, Any]				=   state_dict.pop(snake_case_  )
																					_lowercase							:		Tuple				=   val
														if "input_proj" in key:
																					_lowercase							:		Any				=   state_dict.pop(snake_case_  )
																					_lowercase							:		Optional[Any]				=   val
														if "level_embed" in key or "pos_trans" in key or "pix_trans" in key or "enc_output" in key:
																					_lowercase							:		List[Any]				=   state_dict.pop(snake_case_  )
																					_lowercase							:		Union[str, Any]				=   val

    # finally, create HuggingFace model and load state dict
							_lowercase							:		Any				=   DetaForObjectDetection(snake_case_  )
							model.load_state_dict(snake_case_  )
							model.eval()

							_lowercase							:		Optional[Any]				=   '''cuda''' if torch.cuda.is_available() else '''cpu'''
							model.to(snake_case_  )

							# load image processor
							_lowercase							:		Tuple				=   DetaImageProcessor(format='''coco_detection'''  )

							# verify our conversion on image
							_lowercase							:		Dict				=   prepare_img()
							_lowercase							:		List[Any]				=   processor(images=snake_case_ ,       return_tensors='''pt'''  )
							_lowercase							:		Dict				=   encoding['''pixel_values''']
							_lowercase							:		int				=   model(pixel_values.to(snake_case_  )  )

							# verify logits
							if model_name == "deta-resnet-50":
														_lowercase							:		Any				=   torch.tensor(
														    [[-7.3_9_7_8, -2.5_4_0_6, -4.1_6_6_8], [-8.2_6_8_4, -3.9_9_3_3, -3.8_0_9_6], [-7.0_5_1_5, -3.7_9_7_3, -5.8_5_1_6]]  )
														_lowercase							:		int				=   torch.tensor([[0.5_0_4_3, 0.4_9_7_3, 0.9_9_9_8], [0.2_5_4_2, 0.5_4_8_9, 0.4_7_4_8], [0.5_4_9_0, 0.2_7_6_5, 0.0_5_7_0]]  )
							elif model_name == "deta-resnet-50-24-epochs":
														_lowercase							:		int				=   torch.tensor(
														    [[-7.1_6_8_8, -2.4_8_5_7, -4.8_6_6_9], [-7.8_6_3_0, -3.8_1_5_4, -4.2_6_7_4], [-7.2_7_3_0, -4.1_8_6_5, -5.5_3_2_3]]  )
														_lowercase							:		Optional[int]				=   torch.tensor([[0.5_0_2_1, 0.4_9_7_1, 0.9_9_9_4], [0.2_5_4_6, 0.5_4_8_6, 0.4_7_3_1], [0.1_6_8_6, 0.1_9_8_6, 0.2_1_4_2]]  )

							assert torch.allclose(outputs.logits[0, :3, :3] ,       expected_logits.to(snake_case_  ) ,       atol=1E-4  )
							assert torch.allclose(outputs.pred_boxes[0, :3, :3] ,       expected_boxes.to(snake_case_  ) ,       atol=1E-4  )
							print('''Everything ok!'''  )

							if pytorch_dump_folder_path:
														# Save model and processor
														logger.info(F"Saving PyTorch model and processor to {pytorch_dump_folder_path}..."  )
														Path(snake_case_  ).mkdir(exist_ok=snake_case_  )
														model.save_pretrained(snake_case_  )
														processor.save_pretrained(snake_case_  )

							# Push to hub
							if push_to_hub:
														print('''Pushing model and processor to hub...'''  )
														model.push_to_hub(F"jozhang97/{model_name}"  )
														processor.push_to_hub(F"jozhang97/{model_name}"  )


if __name__ == "__main__":
						lowerCamelCase__	  =		argparse.ArgumentParser()

						parser.add_argument(
						    '--model_name',
						    type=str,
						    default='deta-resnet-50',
						    choices=['deta-resnet-50', 'deta-resnet-50-24-epochs'],
						    help='Name of the model you\'d like to convert.',
						)
						parser.add_argument(
						    '--pytorch_dump_folder_path',
						    default=None,
						    type=str,
						    help='Path to the folder to output PyTorch model.',
						)
						parser.add_argument(
						    '--push_to_hub', action='store_true', help='Whether or not to push the converted model to the 🤗 hub.'
						)
						lowerCamelCase__	  =		parser.parse_args()
						convert_deta_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)








'''simple docstring'''






import gc
import os
import tempfile
import unittest

import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, StableDiffusionPipeline, UNetaDConditionModel
from diffusers.loaders import AttnProcsLayers, LoraLoaderMixin
from diffusers.models.attention_processor import (
    Attention,
    AttnProcessor,
    AttnProcessor2_0,
    LoRAAttnProcessor,
    LoRAAttnProcessor2_0,
    LoRAXFormersAttnProcessor,
    XFormersAttnProcessor,
)
from diffusers.utils import TEXT_ENCODER_ATTN_MODULE, floats_tensor, torch_device





def 				_SCREAMING_SNAKE_CASE(					snake_case_    :     nn.Module  )		->Any:






							'''simple docstring'''
							_lowercase							:		Optional[Any]				=   {}
							for name in unet.attn_processors.keys():
														_lowercase							:		str				=   None if name.endswith('''attn1.processor'''  ) else unet.config.cross_attention_dim
														if name.startswith('''mid_block'''  ):
																					_lowercase							:		Dict				=   unet.config.block_out_channels[-1]
														elif name.startswith('''up_blocks'''  ):
																					_lowercase							:		Any				=   int(name[len('''up_blocks.'''  )]  )
																					_lowercase							:		Any				=   list(reversed(unet.config.block_out_channels  )  )[block_id]
														elif name.startswith('''down_blocks'''  ):
																					_lowercase							:		int				=   int(name[len('''down_blocks.'''  )]  )
																					_lowercase							:		int				=   unet.config.block_out_channels[block_id]
														_lowercase							:		Any				=   (
														    LoRAAttnProcessor2_0 if hasattr(snake_case_ ,       '''scaled_dot_product_attention'''  ) else LoRAAttnProcessor
														)
														_lowercase							:		Tuple				=   lora_attn_processor_class(
														    hidden_size=snake_case_ ,       cross_attention_dim=snake_case_  )
							_lowercase							:		Any				=   AttnProcsLayers(snake_case_  )
							return lora_attn_procs, unet_lora_layers





def 				_SCREAMING_SNAKE_CASE(					snake_case_    :     nn.Module  )		->List[Any]:






							'''simple docstring'''
							_lowercase							:		int				=   {}
							_lowercase							:		str				=   (
							    LoRAAttnProcessor2_0 if hasattr(snake_case_ ,       '''scaled_dot_product_attention'''  ) else LoRAAttnProcessor
							)
							for name, module in text_encoder.named_modules():
														if name.endswith(snake_case_  ):
																					_lowercase							:		Optional[int]				=   lora_attn_processor_class(
																					    hidden_size=module.out_proj.out_features ,       cross_attention_dim=snake_case_  )
							return text_lora_attn_procs





def 				_SCREAMING_SNAKE_CASE(					snake_case_    :     nn.Module  )		->Union[str, Any]:






							'''simple docstring'''
							_lowercase							:		List[str]				=   create_text_encoder_lora_attn_procs(snake_case_  )
							_lowercase							:		List[Any]				=   AttnProcsLayers(snake_case_  )
							return text_encoder_lora_layers





def 				_SCREAMING_SNAKE_CASE(					snake_case_    :     Dict ,       snake_case_    :     str=False  )		->Optional[int]:






							'''simple docstring'''
							for _, attn_proc in text_lora_attn_procs.items():
														# set up.weights
														for layer_name, layer_module in attn_proc.named_modules():
																					if layer_name.endswith('''_lora'''  ):
																												_lowercase							:		Tuple				=   (
																												    torch.randn_like(layer_module.up.weight  )
																												    if randn_weight
																												    else torch.zeros_like(layer_module.up.weight  )
																												)
																												_lowercase							:		Any				=   torch.nn.Parameter(snake_case_  )




class 	_lowerCAmelCase     (    unittest.TestCase       ):






		'''simple docstring'''
		def 						__lowercase      (						self							:       Dict		)      ->			Tuple:







									'''simple docstring'''





									torch.manual_seed(0		)
									_lowercase							:		Any				=   UNetaDConditionModel(
									    block_out_channels=(32, 64)      ,		layers_per_block=2      ,		sample_size=32      ,		in_channels=4      ,		out_channels=4      ,		down_block_types=('''DownBlock2D''', '''CrossAttnDownBlock2D''')      ,		up_block_types=('''CrossAttnUpBlock2D''', '''UpBlock2D''')      ,		cross_attention_dim=32      ,		)
									_lowercase							:		Optional[Any]				=   DDIMScheduler(
									    beta_start=0.00_085      ,		beta_end=0.012      ,		beta_schedule='''scaled_linear'''      ,		clip_sample=UpperCamelCase_      ,		set_alpha_to_one=UpperCamelCase_      ,		steps_offset=1      ,		)
									torch.manual_seed(0		)
									_lowercase							:		int				=   AutoencoderKL(
									    block_out_channels=[32, 64]      ,		in_channels=3      ,		out_channels=3      ,		down_block_types=['''DownEncoderBlock2D''', '''DownEncoderBlock2D''']      ,		up_block_types=['''UpDecoderBlock2D''', '''UpDecoderBlock2D''']      ,		latent_channels=4      ,		)
									_lowercase							:		Any				=   CLIPTextConfig(
									    bos_token_id=0      ,		eos_token_id=2      ,		hidden_size=32      ,		intermediate_size=37      ,		layer_norm_eps=1e-05      ,		num_attention_heads=4      ,		num_hidden_layers=5      ,		pad_token_id=1      ,		vocab_size=1_000      ,		)
									_lowercase							:		List[Any]				=   CLIPTextModel(UpperCamelCase_		)
									_lowercase							:		Union[str, Any]				=   CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip'''		)

									_lowercase			, _lowercase							:		Optional[int]				=   create_unet_lora_layers(UpperCamelCase_		)
									_lowercase							:		Optional[int]				=   create_text_encoder_lora_layers(UpperCamelCase_		)

									_lowercase							:		Dict				=   {
									    '''unet''': unet,
									    '''scheduler''': scheduler,
									    '''vae''': vae,
									    '''text_encoder''': text_encoder,
									    '''tokenizer''': tokenizer,
									    '''safety_checker''': None,
									    '''feature_extractor''': None,
									}
									_lowercase							:		List[str]				=   {
									    '''unet_lora_layers''': unet_lora_layers,
									    '''text_encoder_lora_layers''': text_encoder_lora_layers,
									    '''unet_lora_attn_procs''': unet_lora_attn_procs,
									}
									return pipeline_components, lora_components
		def 						__lowercase      (						self							:       Optional[Any]		)      ->			Optional[Any]:







									'''simple docstring'''





									_lowercase							:		Optional[Any]				=   1
									_lowercase							:		List[Any]				=   10
									_lowercase							:		Union[str, Any]				=   4
									_lowercase							:		Union[str, Any]				=   (32, 32)

									_lowercase							:		Optional[Any]				=   torch.manual_seed(0		)
									_lowercase							:		Tuple				=   floats_tensor((batch_size, num_channels) + sizes		)
									_lowercase							:		str				=   torch.randint(1      ,		UpperCamelCase_      ,		size=(batch_size, sequence_length)      ,		generator=UpperCamelCase_		)

									_lowercase							:		Optional[Any]				=   {
									    '''prompt''': '''A painting of a squirrel eating a burger''',
									    '''generator''': generator,
									    '''num_inference_steps''': 2,
									    '''guidance_scale''': 6.0,
									    '''output_type''': '''numpy''',
									}

									return noise, input_ids, pipeline_inputs

									# copied from: https://colab.research.google.com/gist/sayakpaul/df2ef6e1ae6d8c10a49d859883b10860/scratchpad.ipynb
		def 						__lowercase      (						self							:       Union[str, Any]		)      ->			Any:







									'''simple docstring'''





									_lowercase							:		Optional[Any]				=   77

									_lowercase							:		str				=   torch.randint(2      ,		56      ,		size=(1, max_seq_length)      ,		generator=torch.manual_seed(0		)		)

									_lowercase							:		List[str]				=   {}
									_lowercase							:		Optional[Any]				=   inputs
									return prepared_inputs
		def 						__lowercase      (						self							:       List[str]      ,		UpperCamelCase_							:       str		)      ->			List[str]:







									'''simple docstring'''





									_lowercase			, _lowercase							:		Tuple				=   self.get_dummy_components()
									LoraLoaderMixin.save_lora_weights(
									    save_directory=UpperCamelCase_      ,		unet_lora_layers=lora_components['''unet_lora_layers''']      ,		text_encoder_lora_layers=lora_components['''text_encoder_lora_layers''']      ,		)
									self.assertTrue(os.path.isfile(os.path.join(UpperCamelCase_      ,		'''pytorch_lora_weights.bin'''		)		)		)
		def 						__lowercase      (						self							:       List[str]		)      ->			Union[str, Any]:







									'''simple docstring'''





									_lowercase			, _lowercase							:		Any				=   self.get_dummy_components()
									_lowercase							:		Tuple				=   StableDiffusionPipeline(**UpperCamelCase_		)
									_lowercase							:		Optional[Any]				=   sd_pipe.to(UpperCamelCase_		)
									sd_pipe.set_progress_bar_config(disable=UpperCamelCase_		)

									_lowercase			, _lowercase			, _lowercase							:		Dict				=   self.get_dummy_inputs()

									_lowercase							:		Optional[int]				=   sd_pipe(**UpperCamelCase_		).images
									_lowercase							:		Optional[Any]				=   original_images[0, -3:, -3:, -1]

									with tempfile.TemporaryDirectory() as tmpdirname:
																LoraLoaderMixin.save_lora_weights(
																    save_directory=UpperCamelCase_      ,		unet_lora_layers=lora_components['''unet_lora_layers''']      ,		text_encoder_lora_layers=lora_components['''text_encoder_lora_layers''']      ,		)
																self.assertTrue(os.path.isfile(os.path.join(UpperCamelCase_      ,		'''pytorch_lora_weights.bin'''		)		)		)
																sd_pipe.load_lora_weights(UpperCamelCase_		)

									_lowercase							:		Dict				=   sd_pipe(**UpperCamelCase_		).images
									_lowercase							:		Any				=   lora_images[0, -3:, -3:, -1]

									# Outputs shouldn't match.
									self.assertFalse(torch.allclose(torch.from_numpy(UpperCamelCase_		)      ,		torch.from_numpy(UpperCamelCase_		)		)		)
		def 						__lowercase      (						self							:       List[str]		)      ->			Optional[int]:







									'''simple docstring'''





									_lowercase			, _lowercase							:		Tuple				=   self.get_dummy_components()
									_lowercase							:		int				=   StableDiffusionPipeline(**UpperCamelCase_		)
									_lowercase							:		Dict				=   sd_pipe.to(UpperCamelCase_		)
									sd_pipe.set_progress_bar_config(disable=UpperCamelCase_		)

									_lowercase			, _lowercase			, _lowercase							:		str				=   self.get_dummy_inputs()

									_lowercase							:		Optional[Any]				=   sd_pipe(**UpperCamelCase_		).images
									_lowercase							:		str				=   original_images[0, -3:, -3:, -1]

									with tempfile.TemporaryDirectory() as tmpdirname:
																LoraLoaderMixin.save_lora_weights(
																    save_directory=UpperCamelCase_      ,		unet_lora_layers=lora_components['''unet_lora_layers''']      ,		text_encoder_lora_layers=lora_components['''text_encoder_lora_layers''']      ,		safe_serialization=UpperCamelCase_      ,		)
																self.assertTrue(os.path.isfile(os.path.join(UpperCamelCase_      ,		'''pytorch_lora_weights.safetensors'''		)		)		)
																sd_pipe.load_lora_weights(UpperCamelCase_		)

									_lowercase							:		int				=   sd_pipe(**UpperCamelCase_		).images
									_lowercase							:		int				=   lora_images[0, -3:, -3:, -1]

									# Outputs shouldn't match.
									self.assertFalse(torch.allclose(torch.from_numpy(UpperCamelCase_		)      ,		torch.from_numpy(UpperCamelCase_		)		)		)
		def 						__lowercase      (						self							:       Union[str, Any]		)      ->			Tuple:







									'''simple docstring'''





									_lowercase			, _lowercase							:		str				=   self.get_dummy_components()
									_lowercase							:		Optional[Any]				=   lora_components['''unet_lora_attn_procs''']
									_lowercase							:		Optional[int]				=   StableDiffusionPipeline(**UpperCamelCase_		)
									_lowercase							:		List[str]				=   sd_pipe.to(UpperCamelCase_		)
									sd_pipe.set_progress_bar_config(disable=UpperCamelCase_		)

									_lowercase			, _lowercase			, _lowercase							:		Any				=   self.get_dummy_inputs()

									_lowercase							:		Any				=   sd_pipe(**UpperCamelCase_		).images
									_lowercase							:		Union[str, Any]				=   original_images[0, -3:, -3:, -1]

									with tempfile.TemporaryDirectory() as tmpdirname:
																_lowercase							:		Union[str, Any]				=   sd_pipe.unet
																unet.set_attn_processor(UpperCamelCase_		)
																unet.save_attn_procs(UpperCamelCase_		)
																self.assertTrue(os.path.isfile(os.path.join(UpperCamelCase_      ,		'''pytorch_lora_weights.bin'''		)		)		)
																sd_pipe.load_lora_weights(UpperCamelCase_		)

									_lowercase							:		List[str]				=   sd_pipe(**UpperCamelCase_		).images
									_lowercase							:		Tuple				=   lora_images[0, -3:, -3:, -1]

									# Outputs shouldn't match.
									self.assertFalse(torch.allclose(torch.from_numpy(UpperCamelCase_		)      ,		torch.from_numpy(UpperCamelCase_		)		)		)
		def 						__lowercase      (						self							:       Union[str, Any]		)      ->			Tuple:







									'''simple docstring'''





									_lowercase			, _lowercase							:		Union[str, Any]				=   self.get_dummy_components()
									_lowercase							:		Any				=   StableDiffusionPipeline(**UpperCamelCase_		)

									_lowercase							:		int				=   self.get_dummy_tokens()

									# inference without lora
									_lowercase							:		Dict				=   pipe.text_encoder(**UpperCamelCase_		)[0]
									assert outputs_without_lora.shape == (1, 77, 32)

									# create lora_attn_procs with zeroed out up.weights
									_lowercase							:		Any				=   create_text_encoder_lora_attn_procs(pipe.text_encoder		)
									set_lora_up_weights(UpperCamelCase_      ,		randn_weight=UpperCamelCase_		)

									# monkey patch
									pipe._modify_text_encoder(UpperCamelCase_		)

									# verify that it's okay to release the text_attn_procs which holds the LoRAAttnProcessor.
									del text_attn_procs
									gc.collect()

									# inference with lora
									_lowercase							:		Dict				=   pipe.text_encoder(**UpperCamelCase_		)[0]
									assert outputs_with_lora.shape == (1, 77, 32)

									assert torch.allclose(
									    UpperCamelCase_      ,		UpperCamelCase_		), "lora_up_weight are all zero, so the lora outputs should be the same to without lora outputs"

									# create lora_attn_procs with randn up.weights
									_lowercase							:		str				=   create_text_encoder_lora_attn_procs(pipe.text_encoder		)
									set_lora_up_weights(UpperCamelCase_      ,		randn_weight=UpperCamelCase_		)

									# monkey patch
									pipe._modify_text_encoder(UpperCamelCase_		)

									# verify that it's okay to release the text_attn_procs which holds the LoRAAttnProcessor.
									del text_attn_procs
									gc.collect()

									# inference with lora
									_lowercase							:		List[str]				=   pipe.text_encoder(**UpperCamelCase_		)[0]
									assert outputs_with_lora.shape == (1, 77, 32)

									assert not torch.allclose(
									    UpperCamelCase_      ,		UpperCamelCase_		), "lora_up_weight are not zero, so the lora outputs should be different to without lora outputs"
		def 						__lowercase      (						self							:       List[Any]		)      ->			Union[str, Any]:







									'''simple docstring'''





									_lowercase			, _lowercase							:		Optional[Any]				=   self.get_dummy_components()
									_lowercase							:		int				=   StableDiffusionPipeline(**UpperCamelCase_		)

									_lowercase							:		Tuple				=   self.get_dummy_tokens()

									# inference without lora
									_lowercase							:		Any				=   pipe.text_encoder(**UpperCamelCase_		)[0]
									assert outputs_without_lora.shape == (1, 77, 32)

									# create lora_attn_procs with randn up.weights
									_lowercase							:		Tuple				=   create_text_encoder_lora_attn_procs(pipe.text_encoder		)
									set_lora_up_weights(UpperCamelCase_      ,		randn_weight=UpperCamelCase_		)

									# monkey patch
									pipe._modify_text_encoder(UpperCamelCase_		)

									# verify that it's okay to release the text_attn_procs which holds the LoRAAttnProcessor.
									del text_attn_procs
									gc.collect()

									# inference with lora
									_lowercase							:		Tuple				=   pipe.text_encoder(**UpperCamelCase_		)[0]
									assert outputs_with_lora.shape == (1, 77, 32)

									assert not torch.allclose(
									    UpperCamelCase_      ,		UpperCamelCase_		), "lora outputs should be different to without lora outputs"

									# remove monkey patch
									pipe._remove_text_encoder_monkey_patch()

									# inference with removed lora
									_lowercase							:		Dict				=   pipe.text_encoder(**UpperCamelCase_		)[0]
									assert outputs_without_lora_removed.shape == (1, 77, 32)

									assert torch.allclose(
									    UpperCamelCase_      ,		UpperCamelCase_		), "remove lora monkey patch should restore the original outputs"
		def 						__lowercase      (						self							:       Union[str, Any]		)      ->			Optional[Any]:







									'''simple docstring'''





									_lowercase			, _lowercase							:		str				=   self.get_dummy_components()
									_lowercase							:		Optional[Any]				=   StableDiffusionPipeline(**UpperCamelCase_		)
									_lowercase							:		str				=   sd_pipe.to(UpperCamelCase_		)
									sd_pipe.set_progress_bar_config(disable=UpperCamelCase_		)

									_lowercase			, _lowercase			, _lowercase							:		str				=   self.get_dummy_inputs()

									with tempfile.TemporaryDirectory() as tmpdirname:
																LoraLoaderMixin.save_lora_weights(
																    save_directory=UpperCamelCase_      ,		unet_lora_layers=lora_components['''unet_lora_layers''']      ,		text_encoder_lora_layers=lora_components['''text_encoder_lora_layers''']      ,		)
																self.assertTrue(os.path.isfile(os.path.join(UpperCamelCase_      ,		'''pytorch_lora_weights.bin'''		)		)		)
																sd_pipe.load_lora_weights(UpperCamelCase_		)

									_lowercase							:		Dict				=   sd_pipe(**UpperCamelCase_		).images
									_lowercase							:		List[Any]				=   lora_images[0, -3:, -3:, -1]

									_lowercase							:		List[str]				=   sd_pipe(**UpperCamelCase_      ,		cross_attention_kwargs={'''scale''': 0.5}		).images
									_lowercase							:		Tuple				=   lora_images_with_scale[0, -3:, -3:, -1]

									# Outputs shouldn't match.
									self.assertFalse(
									    torch.allclose(torch.from_numpy(UpperCamelCase_		)      ,		torch.from_numpy(UpperCamelCase_		)		)		)
		def 						__lowercase      (						self							:       str		)      ->			Optional[int]:







									'''simple docstring'''





									with tempfile.TemporaryDirectory() as tmpdirname:
																self.create_lora_weight_file(UpperCamelCase_		)

																_lowercase			, _lowercase							:		Union[str, Any]				=   self.get_dummy_components()
																_lowercase							:		Tuple				=   StableDiffusionPipeline(**UpperCamelCase_		)
																_lowercase							:		List[Any]				=   sd_pipe.to(UpperCamelCase_		)
																sd_pipe.set_progress_bar_config(disable=UpperCamelCase_		)

																# check if vanilla attention processors are used
																for _, module in sd_pipe.unet.named_modules():
																							if isinstance(UpperCamelCase_      ,		UpperCamelCase_		):
																														self.assertIsInstance(module.processor      ,		(AttnProcessor, AttnProcessor2_0)		)

            # load LoRA weight file
																sd_pipe.load_lora_weights(UpperCamelCase_		)

																# check if lora attention processors are used
																for _, module in sd_pipe.unet.named_modules():
																							if isinstance(UpperCamelCase_      ,		UpperCamelCase_		):
																														_lowercase							:		Optional[Any]				=   (
																														    LoRAAttnProcessor2_0 if hasattr(UpperCamelCase_      ,		'''scaled_dot_product_attention'''		) else LoRAAttnProcessor
																														)
																														self.assertIsInstance(module.processor      ,		UpperCamelCase_		)
		@unittest.skipIf(torch_device != '''cuda'''      ,		'''This test is supposed to run on GPU'''		)
		def 						__lowercase      (						self							:       str		)      ->			Tuple:







									'''simple docstring'''





									with tempfile.TemporaryDirectory() as tmpdirname:
																self.create_lora_weight_file(UpperCamelCase_		)

																_lowercase			, _lowercase							:		Dict				=   self.get_dummy_components()
																_lowercase							:		Dict				=   StableDiffusionPipeline(**UpperCamelCase_		)
																_lowercase							:		str				=   sd_pipe.to(UpperCamelCase_		)
																sd_pipe.set_progress_bar_config(disable=UpperCamelCase_		)

																# enable XFormers
																sd_pipe.enable_xformers_memory_efficient_attention()

																# check if xFormers attention processors are used
																for _, module in sd_pipe.unet.named_modules():
																							if isinstance(UpperCamelCase_      ,		UpperCamelCase_		):
																														self.assertIsInstance(module.processor      ,		UpperCamelCase_		)

            # load LoRA weight file
																sd_pipe.load_lora_weights(UpperCamelCase_		)

																# check if lora attention processors are used
																for _, module in sd_pipe.unet.named_modules():
																							if isinstance(UpperCamelCase_      ,		UpperCamelCase_		):
																														self.assertIsInstance(module.processor      ,		UpperCamelCase_		)





		@unittest.skipIf(torch_device != '''cuda'''      ,		'''This test is supposed to run on GPU'''		)
		def 						__lowercase      (						self							:       List[str]		)      ->			str:







									'''simple docstring'''





									_lowercase			, _lowercase							:		Dict				=   self.get_dummy_components()
									_lowercase							:		Dict				=   StableDiffusionPipeline(**UpperCamelCase_		)
									_lowercase							:		Union[str, Any]				=   sd_pipe.to(UpperCamelCase_		)
									sd_pipe.set_progress_bar_config(disable=UpperCamelCase_		)

									_lowercase			, _lowercase			, _lowercase							:		Dict				=   self.get_dummy_inputs()

									# enable XFormers
									sd_pipe.enable_xformers_memory_efficient_attention()

									_lowercase							:		Union[str, Any]				=   sd_pipe(**UpperCamelCase_		).images
									_lowercase							:		Dict				=   original_images[0, -3:, -3:, -1]

									with tempfile.TemporaryDirectory() as tmpdirname:
																LoraLoaderMixin.save_lora_weights(
																    save_directory=UpperCamelCase_      ,		unet_lora_layers=lora_components['''unet_lora_layers''']      ,		text_encoder_lora_layers=lora_components['''text_encoder_lora_layers''']      ,		)
																self.assertTrue(os.path.isfile(os.path.join(UpperCamelCase_      ,		'''pytorch_lora_weights.bin'''		)		)		)
																sd_pipe.load_lora_weights(UpperCamelCase_		)

									_lowercase							:		List[str]				=   sd_pipe(**UpperCamelCase_		).images
									_lowercase							:		List[Any]				=   lora_images[0, -3:, -3:, -1]

									# Outputs shouldn't match.
									self.assertFalse(torch.allclose(torch.from_numpy(UpperCamelCase_		)      ,		torch.from_numpy(UpperCamelCase_		)		)		)



