



"""simple docstring"""
import random
import unittest

import numpy as np

from diffusers import (
    DPMSolverMultistepScheduler,
    EulerAncestralDiscreteScheduler,
    EulerDiscreteScheduler,
    LMSDiscreteScheduler,
    OnnxStableDiffusionImgaImgPipeline,
    PNDMScheduler,
)
from diffusers.utils import floats_tensor
from diffusers.utils.testing_utils import (
    is_onnx_available,
    load_image,
    nightly,
    require_onnxruntime,
    require_torch_gpu,
)

from ..test_pipelines_onnx_common import OnnxPipelineTesterMixin


if is_onnx_available():
      import onnxruntime as ort



class       SCREAMING_SNAKE_CASE__  (					UpperCAmelCase__      ,			unittest.TestCase ):
   __lowerCAmelCase  :   List[str]						=						'hf-internal-testing/tiny-random-OnnxStableDiffusionPipeline'

   def 	SCREAMING_SNAKE_CASE     (   self      ,     _SCREAMING_SNAKE_CASE=0						)  ->      int:





    '''simple docstring'''




    UpperCAmelCase	:					Any	=	floats_tensor((1, 3, 128, 128)      ,     rng=random.Random(_SCREAMING_SNAKE_CASE						)						)
    UpperCAmelCase	:					Optional[int]	=	np.random.RandomState(_SCREAMING_SNAKE_CASE						)
    UpperCAmelCase	:					Union[str, Any]	=	{
        """prompt""": """A painting of a squirrel eating a burger""",
        """image""": image,
        """generator""": generator,
        """num_inference_steps""": 3,
        """strength""": 0.75,
        """guidance_scale""": 7.5,
        """output_type""": """numpy""",
    }
    return inputs

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Tuple:





    '''simple docstring'''




    UpperCAmelCase	:					List[Any]	=	OnnxStableDiffusionImgaImgPipeline.from_pretrained(self.hub_checkpoint      ,     provider="""CPUExecutionProvider"""						)
    pipe.set_progress_bar_config(disable=_SCREAMING_SNAKE_CASE						)

    UpperCAmelCase	:					Optional[int]	=	self.get_dummy_inputs()
    UpperCAmelCase	:					Union[str, Any]	=	pipe(**_SCREAMING_SNAKE_CASE						).images
    UpperCAmelCase	:					int	=	image[0, -3:, -3:, -1].flatten()

    assert image.shape == (1, 128, 128, 3)
    UpperCAmelCase	:					Union[str, Any]	=	np.array([0.6_9643, 0.5_8484, 0.5_0314, 0.5_8760, 0.5_5368, 0.5_9643, 0.5_1529, 0.4_1217, 0.4_9087]						)
    assert np.abs(image_slice - expected_slice						).max() < 1E-1

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      int:





    '''simple docstring'''




    UpperCAmelCase	:					List[str]	=	OnnxStableDiffusionImgaImgPipeline.from_pretrained(self.hub_checkpoint      ,     provider="""CPUExecutionProvider"""						)
    UpperCAmelCase	:					Optional[Any]	=	PNDMScheduler.from_config(pipe.scheduler.config      ,     skip_prk_steps=_SCREAMING_SNAKE_CASE						)
    pipe.set_progress_bar_config(disable=_SCREAMING_SNAKE_CASE						)

    UpperCAmelCase	:					Optional[int]	=	self.get_dummy_inputs()
    UpperCAmelCase	:					Optional[Any]	=	pipe(**_SCREAMING_SNAKE_CASE						).images
    UpperCAmelCase	:					int	=	image[0, -3:, -3:, -1]

    assert image.shape == (1, 128, 128, 3)
    UpperCAmelCase	:					List[Any]	=	np.array([0.6_1737, 0.5_4642, 0.5_3183, 0.5_4465, 0.5_2742, 0.6_0525, 0.4_9969, 0.4_0655, 0.4_8154]						)

    assert np.abs(image_slice.flatten() - expected_slice						).max() < 1E-1

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Tuple:





    '''simple docstring'''




    UpperCAmelCase	:					int	=	OnnxStableDiffusionImgaImgPipeline.from_pretrained(self.hub_checkpoint      ,     provider="""CPUExecutionProvider"""						)
    UpperCAmelCase	:					str	=	LMSDiscreteScheduler.from_config(pipe.scheduler.config						)
    pipe.set_progress_bar_config(disable=_SCREAMING_SNAKE_CASE						)

    # warmup pass to apply optimizations
    UpperCAmelCase	:					Union[str, Any]	=	pipe(**self.get_dummy_inputs()						)

    UpperCAmelCase	:					Optional[int]	=	self.get_dummy_inputs()
    UpperCAmelCase	:					Tuple	=	pipe(**_SCREAMING_SNAKE_CASE						).images
    UpperCAmelCase	:					List[Any]	=	image[0, -3:, -3:, -1]

    assert image.shape == (1, 128, 128, 3)
    UpperCAmelCase	:					Optional[int]	=	np.array([0.5_2761, 0.5_9977, 0.4_9033, 0.4_9619, 0.5_4282, 0.5_0311, 0.4_7600, 0.4_0918, 0.4_5203]						)

    assert np.abs(image_slice.flatten() - expected_slice						).max() < 1E-1

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      str:





    '''simple docstring'''




    UpperCAmelCase	:					List[Any]	=	OnnxStableDiffusionImgaImgPipeline.from_pretrained(self.hub_checkpoint      ,     provider="""CPUExecutionProvider"""						)
    UpperCAmelCase	:					Union[str, Any]	=	EulerDiscreteScheduler.from_config(pipe.scheduler.config						)
    pipe.set_progress_bar_config(disable=_SCREAMING_SNAKE_CASE						)

    UpperCAmelCase	:					List[str]	=	self.get_dummy_inputs()
    UpperCAmelCase	:					Any	=	pipe(**_SCREAMING_SNAKE_CASE						).images
    UpperCAmelCase	:					Dict	=	image[0, -3:, -3:, -1]

    assert image.shape == (1, 128, 128, 3)
    UpperCAmelCase	:					int	=	np.array([0.5_2911, 0.6_0004, 0.4_9229, 0.4_9805, 0.5_4502, 0.5_0680, 0.4_7777, 0.4_1028, 0.4_5304]						)

    assert np.abs(image_slice.flatten() - expected_slice						).max() < 1E-1

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Optional[Any]:





    '''simple docstring'''




    UpperCAmelCase	:					Dict	=	OnnxStableDiffusionImgaImgPipeline.from_pretrained(self.hub_checkpoint      ,     provider="""CPUExecutionProvider"""						)
    UpperCAmelCase	:					Optional[int]	=	EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config						)
    pipe.set_progress_bar_config(disable=_SCREAMING_SNAKE_CASE						)

    UpperCAmelCase	:					Union[str, Any]	=	self.get_dummy_inputs()
    UpperCAmelCase	:					List[Any]	=	pipe(**_SCREAMING_SNAKE_CASE						).images
    UpperCAmelCase	:					Tuple	=	image[0, -3:, -3:, -1]

    assert image.shape == (1, 128, 128, 3)
    UpperCAmelCase	:					List[Any]	=	np.array([0.5_2911, 0.6_0004, 0.4_9229, 0.4_9805, 0.5_4502, 0.5_0680, 0.4_7777, 0.4_1028, 0.4_5304]						)

    assert np.abs(image_slice.flatten() - expected_slice						).max() < 1E-1




   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Optional[int]:





    '''simple docstring'''




    UpperCAmelCase	:					Optional[int]	=	OnnxStableDiffusionImgaImgPipeline.from_pretrained(self.hub_checkpoint      ,     provider="""CPUExecutionProvider"""						)
    UpperCAmelCase	:					List[str]	=	DPMSolverMultistepScheduler.from_config(pipe.scheduler.config						)
    pipe.set_progress_bar_config(disable=_SCREAMING_SNAKE_CASE						)

    UpperCAmelCase	:					List[Any]	=	self.get_dummy_inputs()
    UpperCAmelCase	:					str	=	pipe(**_SCREAMING_SNAKE_CASE						).images
    UpperCAmelCase	:					Union[str, Any]	=	image[0, -3:, -3:, -1]

    assert image.shape == (1, 128, 128, 3)
    UpperCAmelCase	:					Dict	=	np.array([0.6_5331, 0.5_8277, 0.4_8204, 0.5_6059, 0.5_3665, 0.5_6235, 0.5_0969, 0.4_0009, 0.4_6552]						)

    assert np.abs(image_slice.flatten() - expected_slice						).max() < 1E-1



@nightly
@require_onnxruntime
@require_torch_gpu
class       SCREAMING_SNAKE_CASE__  (					unittest.TestCase ):

   @property
   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Tuple:





    '''simple docstring'''




    return (
        "CUDAExecutionProvider",
        {
            "gpu_mem_limit": "15000000000",  # 15GB
            "arena_extend_strategy": "kSameAsRequested",
        },
    )

   @property
   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Any:





    '''simple docstring'''




    UpperCAmelCase	:					List[str]	=	ort.SessionOptions()
    UpperCAmelCase	:					str	=	False
    return options

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Optional[int]:





    '''simple docstring'''




    UpperCAmelCase	:					Dict	=	load_image(
        """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"""
        """/img2img/sketch-mountains-input.jpg"""						)
    UpperCAmelCase	:					Tuple	=	init_image.resize((768, 512)						)
    # using the PNDM scheduler by default
    UpperCAmelCase	:					str	=	OnnxStableDiffusionImgaImgPipeline.from_pretrained(
        """CompVis/stable-diffusion-v1-4"""      ,     revision="""onnx"""      ,     safety_checker=_SCREAMING_SNAKE_CASE      ,     feature_extractor=_SCREAMING_SNAKE_CASE      ,     provider=self.gpu_provider      ,     sess_options=self.gpu_options      ,     )
    pipe.set_progress_bar_config(disable=_SCREAMING_SNAKE_CASE						)

    UpperCAmelCase	:					Any	=	"""A fantasy landscape, trending on artstation"""

    UpperCAmelCase	:					str	=	np.random.RandomState(0						)
    UpperCAmelCase	:					str	=	pipe(
        prompt=_SCREAMING_SNAKE_CASE      ,     image=_SCREAMING_SNAKE_CASE      ,     strength=0.75      ,     guidance_scale=7.5      ,     num_inference_steps=10      ,     generator=_SCREAMING_SNAKE_CASE      ,     output_type="""np"""      ,     )
    UpperCAmelCase	:					Tuple	=	output.images
    UpperCAmelCase	:					List[Any]	=	images[0, 255:258, 383:386, -1]

    assert images.shape == (1, 512, 768, 3)
    UpperCAmelCase	:					Any	=	np.array([0.4909, 0.5059, 0.5372, 0.4623, 0.4876, 0.5049, 0.4820, 0.4956, 0.5019]						)
    # TODO: lower the tolerance after finding the cause of onnxruntime reproducibility issues

    assert np.abs(image_slice.flatten() - expected_slice						).max() < 2E-2




   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Optional[Any]:





    '''simple docstring'''




    UpperCAmelCase	:					Optional[Any]	=	load_image(
        """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"""
        """/img2img/sketch-mountains-input.jpg"""						)
    UpperCAmelCase	:					int	=	init_image.resize((768, 512)						)
    UpperCAmelCase	:					int	=	LMSDiscreteScheduler.from_pretrained(
        """runwayml/stable-diffusion-v1-5"""      ,     subfolder="""scheduler"""      ,     revision="""onnx"""						)
    UpperCAmelCase	:					Any	=	OnnxStableDiffusionImgaImgPipeline.from_pretrained(
        """runwayml/stable-diffusion-v1-5"""      ,     revision="""onnx"""      ,     scheduler=_SCREAMING_SNAKE_CASE      ,     safety_checker=_SCREAMING_SNAKE_CASE      ,     feature_extractor=_SCREAMING_SNAKE_CASE      ,     provider=self.gpu_provider      ,     sess_options=self.gpu_options      ,     )
    pipe.set_progress_bar_config(disable=_SCREAMING_SNAKE_CASE						)

    UpperCAmelCase	:					Union[str, Any]	=	"""A fantasy landscape, trending on artstation"""

    UpperCAmelCase	:					List[str]	=	np.random.RandomState(0						)
    UpperCAmelCase	:					Any	=	pipe(
        prompt=_SCREAMING_SNAKE_CASE      ,     image=_SCREAMING_SNAKE_CASE      ,     strength=0.75      ,     guidance_scale=7.5      ,     num_inference_steps=20      ,     generator=_SCREAMING_SNAKE_CASE      ,     output_type="""np"""      ,     )
    UpperCAmelCase	:					List[str]	=	output.images
    UpperCAmelCase	:					Dict	=	images[0, 255:258, 383:386, -1]

    assert images.shape == (1, 512, 768, 3)
    UpperCAmelCase	:					Tuple	=	np.array([0.8043, 0.926, 0.9581, 0.8119, 0.8954, 0.913, 0.7209, 0.7463, 0.7431]						)
    # TODO: lower the tolerance after finding the cause of onnxruntime reproducibility issues

    assert np.abs(image_slice.flatten() - expected_slice						).max() < 2E-2








'''simple docstring'''







import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import (
    BertTokenizer,
    ViltConfig,
    ViltForImageAndTextRetrieval,
    ViltForImagesAndTextClassification,
    ViltForMaskedLM,
    ViltForQuestionAnswering,
    ViltImageProcessor,
    ViltProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
_A	:			List[str]         =logging.get_logger(__name__)





def  SCREAMING_SNAKE_CASE_       (UpperCamelCase				,			UpperCamelCase=False				,			UpperCamelCase=False				,			UpperCamelCase=False      )      ->       Union[str, Any]:
							lowerCamelCase__					:	str         =       []
							for i in range(config.num_hidden_layers      ):
														# encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
														rename_keys.append((f'''transformer.blocks.{i}.norm1.weight''', f'''vilt.encoder.layer.{i}.layernorm_before.weight''')      )
														rename_keys.append((f'''transformer.blocks.{i}.norm1.bias''', f'''vilt.encoder.layer.{i}.layernorm_before.bias''')      )
														rename_keys.append(
														    (f'''transformer.blocks.{i}.attn.proj.weight''', f'''vilt.encoder.layer.{i}.attention.output.dense.weight''')      )
														rename_keys.append(
														    (f'''transformer.blocks.{i}.attn.proj.bias''', f'''vilt.encoder.layer.{i}.attention.output.dense.bias''')      )
														rename_keys.append((f'''transformer.blocks.{i}.norm2.weight''', f'''vilt.encoder.layer.{i}.layernorm_after.weight''')      )
														rename_keys.append((f'''transformer.blocks.{i}.norm2.bias''', f'''vilt.encoder.layer.{i}.layernorm_after.bias''')      )
														rename_keys.append(
														    (f'''transformer.blocks.{i}.mlp.fc1.weight''', f'''vilt.encoder.layer.{i}.intermediate.dense.weight''')      )
														rename_keys.append((f'''transformer.blocks.{i}.mlp.fc1.bias''', f'''vilt.encoder.layer.{i}.intermediate.dense.bias''')      )
														rename_keys.append((f'''transformer.blocks.{i}.mlp.fc2.weight''', f'''vilt.encoder.layer.{i}.output.dense.weight''')      )
														rename_keys.append((f'''transformer.blocks.{i}.mlp.fc2.bias''', f'''vilt.encoder.layer.{i}.output.dense.bias''')      )

							# embeddings
							rename_keys.extend(
							    [
							        # text embeddings
							        ("""text_embeddings.word_embeddings.weight""", """vilt.embeddings.text_embeddings.word_embeddings.weight"""),
							        (
							            """text_embeddings.position_embeddings.weight""",
							            """vilt.embeddings.text_embeddings.position_embeddings.weight""",
							        ),
							        ("""text_embeddings.position_ids""", """vilt.embeddings.text_embeddings.position_ids"""),
							        (
							            """text_embeddings.token_type_embeddings.weight""",
							            """vilt.embeddings.text_embeddings.token_type_embeddings.weight""",
							        ),
							        ("""text_embeddings.LayerNorm.weight""", """vilt.embeddings.text_embeddings.LayerNorm.weight"""),
							        ("""text_embeddings.LayerNorm.bias""", """vilt.embeddings.text_embeddings.LayerNorm.bias"""),
							        # patch embeddings
							        ("""transformer.cls_token""", """vilt.embeddings.cls_token"""),
							        ("""transformer.patch_embed.proj.weight""", """vilt.embeddings.patch_embeddings.projection.weight"""),
							        ("""transformer.patch_embed.proj.bias""", """vilt.embeddings.patch_embeddings.projection.bias"""),
							        ("""transformer.pos_embed""", """vilt.embeddings.position_embeddings"""),
							        # token type embeddings
							        ("""token_type_embeddings.weight""", """vilt.embeddings.token_type_embeddings.weight"""),
							    ]      )

							# final layernorm + pooler
							rename_keys.extend(
							    [
							        ("""transformer.norm.weight""", """vilt.layernorm.weight"""),
							        ("""transformer.norm.bias""", """vilt.layernorm.bias"""),
							        ("""pooler.dense.weight""", """vilt.pooler.dense.weight"""),
							        ("""pooler.dense.bias""", """vilt.pooler.dense.bias"""),
							    ]      )

							# classifier head(s)
							if vqa_model:
														# classification head
														rename_keys.extend(
														    [
														        ("""vqa_classifier.0.weight""", """classifier.0.weight"""),
														        ("""vqa_classifier.0.bias""", """classifier.0.bias"""),
														        ("""vqa_classifier.1.weight""", """classifier.1.weight"""),
														        ("""vqa_classifier.1.bias""", """classifier.1.bias"""),
														        ("""vqa_classifier.3.weight""", """classifier.3.weight"""),
														        ("""vqa_classifier.3.bias""", """classifier.3.bias"""),
														    ]      )
							elif nlvr_model:
														# classification head
														rename_keys.extend(
														    [
														        ("""nlvr2_classifier.0.weight""", """classifier.0.weight"""),
														        ("""nlvr2_classifier.0.bias""", """classifier.0.bias"""),
														        ("""nlvr2_classifier.1.weight""", """classifier.1.weight"""),
														        ("""nlvr2_classifier.1.bias""", """classifier.1.bias"""),
														        ("""nlvr2_classifier.3.weight""", """classifier.3.weight"""),
														        ("""nlvr2_classifier.3.bias""", """classifier.3.bias"""),
														    ]      )
							else:
														pass

							return rename_keys





def  SCREAMING_SNAKE_CASE_       (UpperCamelCase				,			UpperCamelCase      )      ->       Optional[Any]:
							for i in range(config.num_hidden_layers      ):
														lowerCamelCase__					:	str         =       """vilt."""
														# read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
														lowerCamelCase__					:	List[str]         =       state_dict.pop(f'''transformer.blocks.{i}.attn.qkv.weight'''      )
														lowerCamelCase__					:	str         =       state_dict.pop(f'''transformer.blocks.{i}.attn.qkv.bias'''      )
														# next, add query, keys and values (in that order) to the state dict
														lowerCamelCase__					:	Union[str, Any]         =       in_proj_weight[
														    : config.hidden_size, :
														]
														lowerCamelCase__					:	Union[str, Any]         =       in_proj_bias[: config.hidden_size]
														lowerCamelCase__					:	Optional[int]         =       in_proj_weight[
														    config.hidden_size : config.hidden_size * 2, :
														]
														lowerCamelCase__					:	Tuple         =       in_proj_bias[
														    config.hidden_size : config.hidden_size * 2
														]
														lowerCamelCase__					:	Optional[int]         =       in_proj_weight[
														    -config.hidden_size :, :
														]
														lowerCamelCase__					:	List[str]         =       in_proj_bias[-config.hidden_size :]





def  SCREAMING_SNAKE_CASE_       (UpperCamelCase      )      ->       Any:
							lowerCamelCase__					:	Dict         =       ["""head.weight""", """head.bias"""]
							for k in ignore_keys:
														state_dict.pop(UpperCamelCase				,			UpperCamelCase      )





def  SCREAMING_SNAKE_CASE_       (UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase      )      ->       Dict:
							lowerCamelCase__					:	int         =       dct.pop(UpperCamelCase      )
							lowerCamelCase__					:	Optional[Any]         =       val





@torch.no_grad()
def  SCREAMING_SNAKE_CASE_       (UpperCamelCase				,			UpperCamelCase      )      ->       Tuple:
							lowerCamelCase__					:	List[str]         =       ViltConfig(image_size=384				,			patch_size=32				,			tie_word_embeddings=UpperCamelCase      )
							lowerCamelCase__					:	Union[str, Any]         =       False
							lowerCamelCase__					:	List[str]         =       False
							lowerCamelCase__					:	Any         =       False
							lowerCamelCase__					:	int         =       False
							if "vqa" in checkpoint_url:
														lowerCamelCase__					:	List[Any]         =       True
														lowerCamelCase__					:	Any         =       3129
														lowerCamelCase__					:	Tuple         =       """huggingface/label-files"""
														lowerCamelCase__					:	List[str]         =       """vqa2-id2label.json"""
														lowerCamelCase__					:	str         =       json.load(open(hf_hub_download(UpperCamelCase				,			UpperCamelCase				,			repo_type="""dataset"""      )				,			"""r"""      )      )
														lowerCamelCase__					:	Any         =       {int(UpperCamelCase      ): v for k, v in idalabel.items()}
														lowerCamelCase__					:	Union[str, Any]         =       idalabel
														lowerCamelCase__					:	int         =       {v: k for k, v in idalabel.items()}
														lowerCamelCase__					:	Tuple         =       ViltForQuestionAnswering(UpperCamelCase      )
							elif "nlvr" in checkpoint_url:
														lowerCamelCase__					:	Optional[Any]         =       True
														lowerCamelCase__					:	List[Any]         =       2
														lowerCamelCase__					:	Any         =       {0: """False""", 1: """True"""}
														lowerCamelCase__					:	int         =       {v: k for k, v in config.idalabel.items()}
														lowerCamelCase__					:	Any         =       3
														lowerCamelCase__					:	List[str]         =       ViltForImagesAndTextClassification(UpperCamelCase      )
							elif "irtr" in checkpoint_url:
														lowerCamelCase__					:	List[str]         =       True
														lowerCamelCase__					:	Optional[int]         =       ViltForImageAndTextRetrieval(UpperCamelCase      )
							elif "mlm_itm" in checkpoint_url:
														lowerCamelCase__					:	Optional[Any]         =       True
														lowerCamelCase__					:	Optional[Any]         =       ViltForMaskedLM(UpperCamelCase      )
							else:
														raise ValueError("""Unknown model type"""      )

							# load state_dict of original model, remove and rename some keys
							lowerCamelCase__					:	Dict         =       torch.hub.load_state_dict_from_url(UpperCamelCase				,			map_location="""cpu"""      )["""state_dict"""]
							lowerCamelCase__					:	List[Any]         =       create_rename_keys(UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase      )
							for src, dest in rename_keys:
														rename_key(UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase      )
							read_in_q_k_v(UpperCamelCase				,			UpperCamelCase      )
							if mlm_model or irtr_model:
														lowerCamelCase__					:	List[str]         =       ["""itm_score.fc.weight""", """itm_score.fc.bias"""]
														for k in ignore_keys:
																					state_dict.pop(UpperCamelCase				,			UpperCamelCase      )

    # load state dict into HuggingFace model
							model.eval()
							if mlm_model:
														lowerCamelCase__  ,		lowerCamelCase__					:	List[Any]         =       model.load_state_dict(UpperCamelCase				,			strict=UpperCamelCase      )
														assert missing_keys == ["mlm_score.decoder.bias"]
							else:
														model.load_state_dict(UpperCamelCase      )

							# Define processor
							lowerCamelCase__					:	Optional[int]         =       ViltImageProcessor(size=384      )
							lowerCamelCase__					:	List[str]         =       BertTokenizer.from_pretrained("""bert-base-uncased"""      )
							lowerCamelCase__					:	Union[str, Any]         =       ViltProcessor(UpperCamelCase				,			UpperCamelCase      )

							# Forward pass on example inputs (image + text)
							if nlvr_model:
														lowerCamelCase__					:	int         =       Image.open(requests.get("""https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg"""				,			stream=UpperCamelCase      ).raw      )
														lowerCamelCase__					:	int         =       Image.open(requests.get("""https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg"""				,			stream=UpperCamelCase      ).raw      )
														lowerCamelCase__					:	Dict         =       (
														    """The left image contains twice the number of dogs as the right image, and at least two dogs in total are"""
														    """ standing."""
														)
														lowerCamelCase__					:	Optional[int]         =       processor(UpperCamelCase				,			UpperCamelCase				,			return_tensors="""pt"""      )
														lowerCamelCase__					:	Dict         =       processor(UpperCamelCase				,			UpperCamelCase				,			return_tensors="""pt"""      )
														lowerCamelCase__					:	List[str]         =       model(
														    input_ids=encoding_a.input_ids				,			pixel_values=encoding_a.pixel_values				,			pixel_values_a=encoding_a.pixel_values				,			)
							else:
														lowerCamelCase__					:	str         =       Image.open(requests.get("""http://images.cocodataset.org/val2017/000000039769.jpg"""				,			stream=UpperCamelCase      ).raw      )
														if mlm_model:
																					lowerCamelCase__					:	str         =       """a bunch of [MASK] laying on a [MASK]."""
														else:
																					lowerCamelCase__					:	Optional[int]         =       """How many cats are there?"""
														lowerCamelCase__					:	List[str]         =       processor(UpperCamelCase				,			UpperCamelCase				,			return_tensors="""pt"""      )
														lowerCamelCase__					:	Union[str, Any]         =       model(**UpperCamelCase      )

							# Verify outputs
							if mlm_model:
														lowerCamelCase__					:	Tuple         =       torch.Size([1, 11, 30522]      )
														lowerCamelCase__					:	int         =       torch.tensor([-12.5061, -12.5123, -12.5174]      )
														assert outputs.logits.shape == expected_shape
														assert torch.allclose(outputs.logits[0, 0, :3]				,			UpperCamelCase				,			atol=1E-4      )

														# verify masked token prediction equals "cats"
														lowerCamelCase__					:	int         =       outputs.logits[0, 4, :].argmax(-1      ).item()
														assert tokenizer.decode([predicted_id]      ) == "cats"
							elif vqa_model:
														lowerCamelCase__					:	str         =       torch.Size([1, 3129]      )
														lowerCamelCase__					:	Any         =       torch.tensor([-15.9495, -18.1472, -10.3041]      )
														assert torch.allclose(outputs.logits[0, :3]				,			UpperCamelCase				,			atol=1E-4      )
														assert outputs.logits.shape == expected_shape
														assert torch.allclose(outputs.logits[0, 0, :3]				,			UpperCamelCase				,			atol=1E-4      )

														# verify vqa prediction equals "2"
														lowerCamelCase__					:	Tuple         =       outputs.logits.argmax(-1      ).item()
														assert model.config.idalabel[predicted_idx] == "2"
							elif nlvr_model:
														lowerCamelCase__					:	str         =       torch.Size([1, 2]      )
														lowerCamelCase__					:	Optional[Any]         =       torch.tensor([-2.8721, 2.1291]      )
														assert torch.allclose(outputs.logits[0, :3]				,			UpperCamelCase				,			atol=1E-4      )
														assert outputs.logits.shape == expected_shape

							Path(UpperCamelCase      ).mkdir(exist_ok=UpperCamelCase      )
							print(f'''Saving model and processor to {pytorch_dump_folder_path}'''      )
							model.save_pretrained(UpperCamelCase      )
							processor.save_pretrained(UpperCamelCase      )


if __name__ == "__main__":
			_A	:			str         =argparse.ArgumentParser()
			# Required parameters
			parser.add_argument(
			    '''--checkpoint_url''',
			    default='''https://github.com/dandelin/ViLT/releases/download/200k/vilt_200k_mlm_itm.ckpt''',
			    type=str,
			    help='''URL of the checkpoint you\'d like to convert.''',
			)
			parser.add_argument(
			    '''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model directory.'''
			)

			_A	:			Tuple         =parser.parse_args()
			convert_vilt_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)

