
import argparse

import gdown
import numpy as np
import torch
from huggingface_hub import hf_hub_download

from transformers import (
    CLIPTokenizer,
    CLIPTokenizerFast,
    VideoMAEImageProcessor,
    XCLIPConfig,
    XCLIPModel,
    XCLIPProcessor,
    XCLIPTextConfig,
    XCLIPVisionConfig,
)
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		Optional[Any]:
			lowerCAmelCase__      :      int         =		XCLIPTextConfig()

			# derive patch size from model name
			lowerCAmelCase__      :      Union[str, Any]         =		model_name.find('patch'     )
			lowerCAmelCase__      :      List[str]         =		int(model_name[start_idx + len('patch'     ) : start_idx + len('patch'     ) + 2]     )
			lowerCAmelCase__      :      Any         =		XCLIPVisionConfig(patch_size=_lowercase       ,							num_frames=_lowercase     )

			if "large" in model_name:
						lowerCAmelCase__      :      Optional[Any]         =		768
						lowerCAmelCase__      :      Union[str, Any]         =		3_072
						lowerCAmelCase__      :      Any         =		12

						lowerCAmelCase__      :      Any         =		1_024
						lowerCAmelCase__      :      str         =		4_096
						lowerCAmelCase__      :      Union[str, Any]         =		16
						lowerCAmelCase__      :      Union[str, Any]         =		24
						lowerCAmelCase__      :      Tuple         =		768
						lowerCAmelCase__      :      Any         =		3_072

			if model_name == "xclip-large-patch14-16-frames":
						lowerCAmelCase__      :      Any         =		336

			lowerCAmelCase__      :      Any         =		XCLIPConfig.from_text_vision_configs(_lowercase       ,							_lowercase     )

			if "large" in model_name:
						lowerCAmelCase__      :      List[Any]         =		768

			return config
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_     )				->		Optional[int]:


			if name == "token_embedding.weight":
						lowerCAmelCase__      :      Optional[Any]         =		name.replace('token_embedding.weight'       ,							'text_model.embeddings.token_embedding.weight'     )
			if name == "positional_embedding":
						lowerCAmelCase__      :      Tuple         =		name.replace('positional_embedding'       ,							'text_model.embeddings.position_embedding.weight'     )
			if "ln_1" in name:
						lowerCAmelCase__      :      Dict         =		name.replace('ln_1'       ,							'layer_norm1'     )
			if "ln_2" in name:
						lowerCAmelCase__      :      str         =		name.replace('ln_2'       ,							'layer_norm2'     )
			if "c_fc" in name:
						lowerCAmelCase__      :      str         =		name.replace('c_fc'       ,							'fc1'     )
			if "c_proj" in name:
						lowerCAmelCase__      :      int         =		name.replace('c_proj'       ,							'fc2'     )
			if name.startswith('transformer.resblocks'     ):
						lowerCAmelCase__      :      Union[str, Any]         =		name.replace('transformer.resblocks'       ,							'text_model.encoder.layers'     )
			if "attn.out_proj" in name and "message" not in name:
						lowerCAmelCase__      :      Union[str, Any]         =		name.replace('attn.out_proj'       ,							'self_attn.out_proj'     )
			if "ln_final" in name:
						lowerCAmelCase__      :      Union[str, Any]         =		name.replace('ln_final'       ,							'text_model.final_layer_norm'     )
			# visual encoder
			if name == "visual.class_embedding":
						lowerCAmelCase__      :      Any         =		name.replace('visual.class_embedding'       ,							'vision_model.embeddings.class_embedding'     )
			if name == "visual.positional_embedding":
						lowerCAmelCase__      :      Optional[int]         =		name.replace('visual.positional_embedding'       ,							'vision_model.embeddings.position_embedding.weight'     )
			if name.startswith('visual.transformer.resblocks'     ):
						lowerCAmelCase__      :      Union[str, Any]         =		name.replace('visual.transformer.resblocks'       ,							'vision_model.encoder.layers'     )
			if "visual.conv1" in name:
						lowerCAmelCase__      :      int         =		name.replace('visual.conv1'       ,							'vision_model.embeddings.patch_embedding'     )
			if "visual.ln_pre" in name:
						lowerCAmelCase__      :      Any         =		name.replace('visual.ln_pre'       ,							'vision_model.pre_layernorm'     )
			if "visual.ln_post" in name:
						lowerCAmelCase__      :      str         =		name.replace('visual.ln_post'       ,							'vision_model.post_layernorm'     )
			if "visual.proj" in name:
						lowerCAmelCase__      :      Union[str, Any]         =		name.replace('visual.proj'       ,							'visual_projection.weight'     )
			if "text_projection" in name:
						lowerCAmelCase__      :      Dict         =		name.replace('text_projection'       ,							'text_projection.weight'     )
			# things on top
			if "prompts_visual_proj" in name:
						lowerCAmelCase__      :      List[str]         =		name.replace('prompts_visual_proj'       ,							'prompts_visual_projection'     )
			if "prompts_visual_ln" in name:
						lowerCAmelCase__      :      Dict         =		name.replace('prompts_visual_ln'       ,							'prompts_visual_layernorm'     )
			# mit
			if name == "mit.positional_embedding":
						lowerCAmelCase__      :      str         =		name.replace('positional'       ,							'position'     )
			if name.startswith('mit.resblocks'     ):
						lowerCAmelCase__      :      Dict         =		name.replace('mit.resblocks'       ,							'mit.encoder.layers'     )
			# prompts generator
			if name.startswith('prompts_generator.norm'     ):
						lowerCAmelCase__      :      Union[str, Any]         =		name.replace('prompts_generator.norm'       ,							'prompts_generator.layernorm'     )

			return name
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		int:


			for key in orig_state_dict.copy().keys():
						lowerCAmelCase__      :      Dict         =		orig_state_dict.pop(_lowercase     )

						if "attn.in_proj" in key:
									lowerCAmelCase__      :      Optional[Any]         =		key.split('.'     )
									if key.startswith('visual'     ):
												lowerCAmelCase__      :      Any         =		key_split[3]
												lowerCAmelCase__      :      Optional[Any]         =		config.vision_config.hidden_size
												if "message_attn" in key:
															if "weight" in key:
																		lowerCAmelCase__      :      str         =		val[
																		    :dim, :
																		]
																		lowerCAmelCase__      :      Optional[int]         =		val[
																		    dim : dim * 2, :
																		]
																		lowerCAmelCase__      :      Union[str, Any]         =		val[
																		    -dim:, :
																		]
															else:
																		lowerCAmelCase__      :      Dict         =		val[
																		    :dim
																		]
																		lowerCAmelCase__      :      Optional[int]         =		val[
																		    dim : dim * 2
																		]
																		lowerCAmelCase__      :      Optional[int]         =		val[
																		    -dim:
																		]
												else:
															if "weight" in key:
																		lowerCAmelCase__      :      Optional[Any]         =		val[
																		    :dim, :
																		]
																		lowerCAmelCase__      :      List[str]         =		val[
																		    dim : dim * 2, :
																		]
																		lowerCAmelCase__      :      Dict         =		val[
																		    -dim:, :
																		]
															else:
																		lowerCAmelCase__      :      Union[str, Any]         =		val[:dim]
																		lowerCAmelCase__      :      Union[str, Any]         =		val[
																		    dim : dim * 2
																		]
																		lowerCAmelCase__      :      Union[str, Any]         =		val[-dim:]
									elif key.startswith('mit'     ):
												lowerCAmelCase__      :      Tuple         =		key_split[2]
												lowerCAmelCase__      :      Union[str, Any]         =		config.vision_config.mit_hidden_size
												if "weight" in key:
															lowerCAmelCase__      :      Optional[int]         =		val[:dim, :]
															lowerCAmelCase__      :      Optional[int]         =		val[dim : dim * 2, :]
															lowerCAmelCase__      :      str         =		val[-dim:, :]
												else:
															lowerCAmelCase__      :      str         =		val[:dim]
															lowerCAmelCase__      :      Any         =		val[dim : dim * 2]
															lowerCAmelCase__      :      int         =		val[-dim:]
									else:
												lowerCAmelCase__      :      Tuple         =		key_split[2]
												lowerCAmelCase__      :      Any         =		config.text_config.hidden_size
												if "weight" in key:
															lowerCAmelCase__      :      Dict         =		val[:dim, :]
															lowerCAmelCase__      :      Dict         =		val[
															    dim : dim * 2, :
															]
															lowerCAmelCase__      :      List[str]         =		val[-dim:, :]
												else:
															lowerCAmelCase__      :      Any         =		val[:dim]
															lowerCAmelCase__      :      Tuple         =		val[
															    dim : dim * 2
															]
															lowerCAmelCase__      :      List[str]         =		val[-dim:]
						else:
									lowerCAmelCase__      :      Optional[int]         =		rename_key(_lowercase     )
									if new_key_name in ["visual_projection.weight", "text_projection.weight"]:
												lowerCAmelCase__      :      Optional[Any]         =		val.T
									lowerCAmelCase__      :      Tuple         =		val

			return orig_state_dict
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_     )				->		Optional[Any]:


			if num_frames == 8:
						lowerCAmelCase__      :      str         =		"""eating_spaghetti_8_frames.npy"""
			elif num_frames == 16:
						lowerCAmelCase__      :      int         =		"""eating_spaghetti.npy"""
			elif num_frames == 32:
						lowerCAmelCase__      :      List[str]         =		"""eating_spaghetti_32_frames.npy"""
			lowerCAmelCase__      :      int         =		hf_hub_download(
			    repo_id='hf-internal-testing/spaghetti-video'       ,							filename=_lowercase       ,							repo_type='dataset'       ,							)
			lowerCAmelCase__      :      Union[str, Any]         =		np.load(_lowercase     )
			return list(_lowercase     )
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_=None       ,							SCREAMING_SNAKE_CASE_=False     )				->		Dict:
			lowerCAmelCase__      :      List[Any]         =		{
			    # fully supervised kinetics-400 checkpoints
			    """xclip-base-patch32""": """https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k400_32_8.pth""",
			    """xclip-base-patch32-16-frames""": (
			        """https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k400_32_16.pth"""
			    ),
			    """xclip-base-patch16""": """https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k400_16_8.pth""",
			    """xclip-base-patch16-16-frames""": (
			        """https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k400_16_16.pth"""
			    ),
			    """xclip-large-patch14""": """https://drive.google.com/u/0/uc?id=1NUOImq0o5DlQTST17iIP3vG7DgmHQuCx&amp;export=download&amp;confirm=t&amp;uuid=b26caedc-88e2-473e-830a-9d158b653cdb""",
			    """xclip-large-patch14-16-frames""": """https://drive.google.com/u/0/uc?id=1FOYgnJc097OJ4lGwtRCCydQyVPJEOH7d&amp;export=download&amp;confirm=t&amp;uuid=538fa810-e671-4050-b385-9a623f89804f""",
			    # fully supervised kinetics-600 checkpoints
			    """xclip-base-patch16-kinetics-600""": (
			        """https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k600_16_8.pth"""
			    ),
			    """xclip-base-patch16-kinetics-600-16-frames""": (
			        """https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k600_16_16.pth"""
			    ),
			    """xclip-large-patch14-kinetics-600""": """https://drive.google.com/u/0/uc?id=1FV8C1INuM91sLAN4ImjzePLIlpMSihwV&amp;export=download&amp;confirm=t&amp;uuid=141d4977-4a65-44ae-864f-4b0c19f838be""",
			    # few shot
			    """xclip-base-patch16-hmdb-2-shot""": (
			        """https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_hmdb_2.pth"""
			    ),
			    """xclip-base-patch16-hmdb-4-shot""": (
			        """https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_hmdb_4.pth"""
			    ),
			    """xclip-base-patch16-hmdb-8-shot""": (
			        """https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_hmdb_8.pth"""
			    ),
			    """xclip-base-patch16-hmdb-16-shot""": (
			        """https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_hmdb_16.pth"""
			    ),
			    """xclip-base-patch16-ucf-2-shot""": (
			        """https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_ucf_2.pth"""
			    ),
			    """xclip-base-patch16-ucf-4-shot""": (
			        """https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_ucf_4.pth"""
			    ),
			    """xclip-base-patch16-ucf-8-shot""": (
			        """https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_ucf_8.pth"""
			    ),
			    """xclip-base-patch16-ucf-16-shot""": (
			        """https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_ucf_16.pth"""
			    ),
			    # zero shot
			    """xclip-base-patch16-zero-shot""": """https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/zero.pth""",
			}

			lowerCAmelCase__      :      Optional[int]         =		model_to_url[model_name]
			lowerCAmelCase__      :      int         =		8
			if "16-frames" in model_name:
						lowerCAmelCase__      :      List[Any]         =		16
			elif "shot" in model_name:
						lowerCAmelCase__      :      Dict         =		32

			lowerCAmelCase__      :      Optional[int]         =		get_xclip_config(_lowercase       ,							_lowercase     )
			lowerCAmelCase__      :      Optional[Any]         =		XCLIPModel(_lowercase     )
			model.eval()

			if "drive" in checkpoint_url:
						lowerCAmelCase__      :      List[str]         =		"""pytorch_model.bin"""
						gdown.cached_download(_lowercase       ,							_lowercase       ,							quiet=_lowercase     )
						lowerCAmelCase__      :      List[Any]         =		torch.load(_lowercase       ,							map_location='cpu'     )["""model"""]
			else:
						lowerCAmelCase__      :      Tuple         =		torch.hub.load_state_dict_from_url(_lowercase     )["""model"""]

			lowerCAmelCase__      :      Union[str, Any]         =		convert_state_dict(_lowercase       ,							_lowercase     )

			lowerCAmelCase__      :      str         =		XCLIPModel(_lowercase     )
			lowerCAmelCase__      :      Optional[int]         =		model.load_state_dict(_lowercase       ,							strict=_lowercase     )
			assert missing_keys == ["text_model.embeddings.position_ids", "vision_model.embeddings.position_ids"]
			model.eval()

			lowerCAmelCase__      :      List[str]         =		336 if model_name == """xclip-large-patch14-16-frames""" else 224
			lowerCAmelCase__      :      List[Any]         =		VideoMAEImageProcessor(size=_lowercase     )
			lowerCAmelCase__      :      Any         =		CLIPTokenizer.from_pretrained('openai/clip-vit-base-patch32'     )
			lowerCAmelCase__      :      str         =		CLIPTokenizerFast.from_pretrained('openai/clip-vit-base-patch32'     )
			lowerCAmelCase__      :      Optional[Any]         =		XCLIPProcessor(image_processor=_lowercase       ,							tokenizer=_lowercase     )

			lowerCAmelCase__      :      Optional[int]         =		prepare_video(_lowercase     )
			lowerCAmelCase__      :      Optional[Any]         =		processor(
			    text=['playing sports', 'eating spaghetti', 'go shopping']       ,							videos=_lowercase       ,							return_tensors='pt'       ,							padding=_lowercase     )

			print('Shape of pixel values:'       ,							inputs.pixel_values.shape     )

			with torch.no_grad():
						lowerCAmelCase__      :      List[Any]         =		model(**_lowercase     )

			# Verify outputs
			lowerCAmelCase__      :      List[Any]         =		outputs.logits_per_video
			lowerCAmelCase__      :      Any         =		logits_per_video.softmax(dim=1     )
			print('Probs:'       ,							_lowercase     )
			# kinetics-400
			if model_name == "xclip-base-patch32":
						lowerCAmelCase__      :      Union[str, Any]         =		torch.tensor([[0.0019, 0.9951, 0.0030]]     )
			elif model_name == "xclip-base-patch32-16-frames":
						lowerCAmelCase__      :      str         =		torch.tensor([[7.0999e-04, 9.9883e-01, 4.5580e-04]]     )
			elif model_name == "xclip-base-patch16":
						lowerCAmelCase__      :      Tuple         =		torch.tensor([[0.0083, 0.9681, 0.0236]]     )
			elif model_name == "xclip-base-patch16-16-frames":
						lowerCAmelCase__      :      Any         =		torch.tensor([[7.6937e-04, 9.9728e-01, 1.9473e-03]]     )
			elif model_name == "xclip-large-patch14":
						lowerCAmelCase__      :      str         =		torch.tensor([[0.0062, 0.9864, 0.0075]]     )
			elif model_name == "xclip-large-patch14-16-frames":
						lowerCAmelCase__      :      Tuple         =		torch.tensor([[3.3877e-04, 9.9937e-01, 2.8888e-04]]     )
			# kinetics-600
			elif model_name == "xclip-base-patch16-kinetics-600":
						lowerCAmelCase__      :      List[Any]         =		torch.tensor([[0.0555, 0.8914, 0.0531]]     )
			elif model_name == "xclip-base-patch16-kinetics-600-16-frames":
						lowerCAmelCase__      :      Union[str, Any]         =		torch.tensor([[3.8554e-04, 9.9929e-01, 3.2754e-04]]     )
			elif model_name == "xclip-large-patch14-kinetics-600":
						lowerCAmelCase__      :      List[Any]         =		torch.tensor([[0.0036, 0.9920, 0.0045]]     )
			# few shot
			elif model_name == "xclip-base-patch16-hmdb-2-shot":
						lowerCAmelCase__      :      Dict         =		torch.tensor([[7.1890e-06, 9.9994e-01, 5.6559e-05]]     )
			elif model_name == "xclip-base-patch16-hmdb-4-shot":
						lowerCAmelCase__      :      Union[str, Any]         =		torch.tensor([[1.0320e-05, 9.9993e-01, 6.2435e-05]]     )
			elif model_name == "xclip-base-patch16-hmdb-8-shot":
						lowerCAmelCase__      :      str         =		torch.tensor([[4.1377e-06, 9.9990e-01, 9.8386e-05]]     )
			elif model_name == "xclip-base-patch16-hmdb-16-shot":
						lowerCAmelCase__      :      str         =		torch.tensor([[4.1347e-05, 9.9962e-01, 3.3411e-04]]     )
			elif model_name == "xclip-base-patch16-ucf-2-shot":
						lowerCAmelCase__      :      int         =		torch.tensor([[8.5857e-05, 9.9928e-01, 6.3291e-04]]     )
			elif model_name == "xclip-base-patch16-ucf-4-shot":
						lowerCAmelCase__      :      Optional[int]         =		torch.tensor([[8.5857e-05, 9.9928e-01, 6.3291e-04]]     )
			elif model_name == "xclip-base-patch16-ucf-8-shot":
						lowerCAmelCase__      :      Any         =		torch.tensor([[0.0027, 0.9904, 0.0070]]     )
			elif model_name == "xclip-base-patch16-ucf-16-shot":
						lowerCAmelCase__      :      Tuple         =		torch.tensor([[9.8219e-04, 9.9593e-01, 3.0863e-03]]     )
			# zero shot
			elif model_name == "xclip-base-patch16-zero-shot":
						lowerCAmelCase__      :      Union[str, Any]         =		torch.tensor([[3.5082e-04, 9.9785e-01, 1.7966e-03]]     )
			else:
						raise ValueError(F'''Model name {model_name} not supported'''     )
			assert torch.allclose(_lowercase       ,							_lowercase       ,							atol=1e-3     )
			print('Looks ok!'     )

			if pytorch_dump_folder_path is not None:
						print(F'''Saving model {model_name} to {pytorch_dump_folder_path}'''     )
						model.save_pretrained(_lowercase     )

			if push_to_hub:
						print('Pushing model, processor and slow tokenizer files to the hub...'     )
						model.push_to_hub(_lowercase       ,							organization='nielsr'     )
						processor.push_to_hub(_lowercase       ,							organization='nielsr'     )
						slow_tokenizer.push_to_hub(_lowercase       ,							organization='nielsr'     )


if __name__ == "__main__":
					lowerCamelCase__          =		argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    """--model_name""",
					    default="""xclip-base-patch32""",
					    type=str,
					    help="""Name of the model.""",
					)
					parser.add_argument(
					    """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model directory."""
					)
					parser.add_argument(
					    """--push_to_hub""", action="""store_true""", help="""Whether or not to push the converted model to the 🤗 hub."""
					)

					lowerCamelCase__          =		parser.parse_args()
					convert_xclip_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)




"""simple docstring"""
import argparse
import json
import os
from collections import OrderedDict

import numpy as np
import tensorflow as tf
import torch
def    A_  (			_lowercase	):







						'''simple docstring'''


						snake_case_ :Union[str, Any]						=			os.path.join(args.tf_model_dir,					"""parameters.json"""	)
						snake_case_ :Any						=			json.loads(open(_lowercase	).read()	)
						if not params:
												raise ValueError(
												    f"""It seems that the json file at {parameter_file} is empty. Make sure you have a correct json file."""	)
						if not args.output.endswith(""".pt"""	):
												snake_case_ :Optional[int]						=			args.output + """.pt"""
						snake_case_ :List[str]						=			OrderedDict()
						with tf.device("""/CPU:0"""	):
												snake_case_ :Dict						=			tf.train.load_checkpoint(args.tf_model_dir	)
												snake_case_ :str						=			reader.get_variable_to_shape_map()
												for key_name in shapes.keys():
																		snake_case_ :List[Any]						=			reader.get_tensor(_lowercase	).astype(np.floataa	)
																		if key_name.endswith("""/adam_m"""	) or key_name.endswith("""/adam_v"""	):
																								continue
																		if key_name.startswith("""pasts/"""	):
																								if key_name.startswith("""pasts/mlp"""	):
																														snake_case_ :Any						=			int(key_name[9]	)
																								elif key_name.startswith("""pasts/out"""	):
																														snake_case_ :Optional[int]						=			8
																								snake_case_ :List[str]						=			"""model.sqout.%d.weight""" % (player * 2)  # enter to nn.Sequencial with Tanh, so 2 at a time
																								snake_case_ :Optional[Any]						=			vnp.transpose([1, 0]	).copy()  # Mesh-Tensorflow is a diagonal matrix
																								snake_case_ :List[str]						=			torch.tensor(_lowercase	)
																		elif key_name.startswith("""model/moe"""	):
																								snake_case_ :Tuple						=			int(key_name[9:].split("""/"""	)[0]	)
																								if key_name.endswith("""/switch_gating/kernel"""	):
																														snake_case_ :Union[str, Any]						=			"""model.blocks.%d.feed_forward.mlp.router.classifier.weight""" % player
																														snake_case_ :Optional[Any]						=			vnp.transpose([1, 0]	).copy()  # Mesh-Tensorflow is a diagonal matrix
																														snake_case_ :Optional[Any]						=			torch.tensor(_lowercase	)
																								elif key_name.endswith("""/softmlp/kernel"""	):
																														snake_case_ :List[Any]						=			"""model.blocks.%d.feed_forward.soft_bypass_mlp.weight""" % player
																														snake_case_ :Optional[int]						=			vnp.transpose([1, 0]	).copy()  # Mesh-Tensorflow is a diagonal matrix
																														snake_case_ :Optional[Any]						=			torch.tensor(_lowercase	)
																								elif key_name.endswith("""/wo/kernel"""	) or key_name.endswith("""/wi/kernel"""	):
																														snake_case_ :Dict						=			key_name[-9:-7]
																														for i in range(16	):
																																				snake_case_ :str						=			"""model.blocks.%d.feed_forward.mlp.experts.expert_%d.%s.weight""" % (player, i, nlayer)
																																				snake_case_ :Tuple						=			(
																																				    vnp[i].transpose([1, 0]	).copy()
																																				)  # In Mesh-Tensorflow, it is one array, so it is divided
																																				snake_case_ :Optional[int]						=			torch.tensor(_lowercase	)
																		elif key_name.startswith("""model/mlp"""	):
																								snake_case_ :Optional[int]						=			int(key_name[9:].split("""/"""	)[0]	)
																								if key_name.endswith("""/p1/kernel"""	):
																														snake_case_ :Union[str, Any]						=			"""model.blocks.%d.feed_forward.mlp.wi.weight""" % player
																														snake_case_ :Dict						=			vnp.transpose([1, 0]	).copy()  # Mesh-Tensorflow is a diagonal matrix
																														snake_case_ :Optional[Any]						=			torch.tensor(_lowercase	)
																								elif key_name.endswith("""/p1/bias"""	):
																														snake_case_ :List[Any]						=			"""model.blocks.%d.feed_forward.mlp.wi.bias""" % player
																														snake_case_ :str						=			vnp.copy()  # same because it is one dimensional
																														snake_case_ :Optional[Any]						=			torch.tensor(_lowercase	)
																								elif key_name.endswith("""/p2/kernel"""	):
																														snake_case_ :Union[str, Any]						=			"""model.blocks.%d.feed_forward.mlp.wo.weight""" % player
																														snake_case_ :Dict						=			vnp.transpose([1, 0]	).copy()  # Mesh-Tensorflow is a diagonal matrix
																														snake_case_ :Tuple						=			torch.tensor(_lowercase	)
																								elif key_name.endswith("""/p2/bias"""	):
																														snake_case_ :Dict						=			"""model.blocks.%d.feed_forward.mlp.wo.bias""" % player
																														snake_case_ :Any						=			vnp.copy()  # same because it is one dimensional
																														snake_case_ :Optional[int]						=			torch.tensor(_lowercase	)
																		elif key_name.startswith("""model/ln"""	):
																								snake_case_ :Union[str, Any]						=			int(key_name[8:].split("""/"""	)[0]	)
																								if key_name.endswith("""/b"""	):
																														snake_case_ :str						=			"""model.blocks.%d.feed_forward.norm.bias""" % player
																														snake_case_ :Dict						=			vnp.copy()  # same because it is one dimensional
																														snake_case_ :int						=			torch.tensor(_lowercase	)
																								elif key_name.endswith("""/g"""	):
																														snake_case_ :Dict						=			"""model.blocks.%d.feed_forward.norm.weight""" % player
																														snake_case_ :Dict						=			vnp.copy()  # same because it is one dimensional
																														snake_case_ :Tuple						=			torch.tensor(_lowercase	)
																		elif key_name.startswith("""model/att"""	):
																								snake_case_ :List[str]						=			int(key_name[9:].split("""/"""	)[0]	)
																								if key_name.endswith("""/qkv/kernel"""	):
																														snake_case_ :Optional[int]						=			vnp.copy()  # Compute same dimension as Mesh-tensorflow using einsum
																														snake_case_ :Dict						=			state[:, 0, :, :]
																														snake_case_ :int						=			state[:, 1, :, :]
																														snake_case_ :List[str]						=			state[:, 2, :, :]
																														snake_case_ :str						=			(
																														    state_q.reshape([state_q.shape[0], state_q.shape[1] * state_q.shape[2]]	)
																														    .transpose([1, 0]	)
																														    .copy()
																														)  # Mesh-Tensorflow is a diagonal matrix
																														snake_case_ :Any						=			(
																														    state_k.reshape([state_k.shape[0], state_k.shape[1] * state_k.shape[2]]	)
																														    .transpose([1, 0]	)
																														    .copy()
																														)  # Mesh-Tensorflow is a diagonal matrix
																														snake_case_ :Optional[int]						=			(
																														    state_v.reshape([state_v.shape[0], state_v.shape[1] * state_v.shape[2]]	)
																														    .transpose([1, 0]	)
																														    .copy()
																														)  # Mesh-Tensorflow is a diagonal matrix
																														snake_case_ :int						=			"""model.blocks.%d.self_attn.self_attn.q_proj.weight""" % player
																														snake_case_ :int						=			torch.tensor(_lowercase	)
																														snake_case_ :Optional[Any]						=			"""model.blocks.%d.self_attn.self_attn.k_proj.weight""" % player
																														snake_case_ :Dict						=			torch.tensor(_lowercase	)
																														snake_case_ :Dict						=			"""model.blocks.%d.self_attn.self_attn.v_proj.weight""" % player
																														snake_case_ :Optional[Any]						=			torch.tensor(_lowercase	)
																								elif key_name.endswith("""/o/kernel"""	):
																														snake_case_ :str						=			"""model.blocks.%d.self_attn.self_attn.out_proj.weight""" % player
																														snake_case_ :str						=			(
																														    vnp.reshape([vnp.shape[0] * vnp.shape[1], vnp.shape[2]]	).transpose([1, 0]	).copy()
																														)  # Mesh-Tensorflow is a diagonal matrix
																														snake_case_ :Any						=			torch.tensor(_lowercase	)
																		elif key_name.startswith("""model/an"""	):
																								snake_case_ :Optional[int]						=			int(key_name[8:].split("""/"""	)[0]	)
																								if key_name.endswith("""/b"""	):
																														snake_case_ :Any						=			"""model.blocks.%d.self_attn.norm.bias""" % player
																														snake_case_ :Optional[int]						=			vnp.copy()  # same because it is one dimensional
																														snake_case_ :Tuple						=			torch.tensor(_lowercase	)
																								elif key_name.endswith("""/g"""	):
																														snake_case_ :Union[str, Any]						=			"""model.blocks.%d.self_attn.norm.weight""" % player
																														snake_case_ :Dict						=			vnp.copy()  # same because it is one dimensional
																														snake_case_ :Optional[int]						=			torch.tensor(_lowercase	)
																		elif (
																		    key_name.startswith("""model/wte"""	)
																		    or key_name.startswith("""model/wpe"""	)
																		    or key_name.startswith("""model/ete"""	)
																		):
																								snake_case_ :List[Any]						=			{"""wte""": """embed_tokens""", """wpe""": """position_embeddings""", """ete""": """extra_position_embeddings"""}[
																								    key_name[-3:]
																								]
																								snake_case_ :Optional[Any]						=			"""model.%s.weight""" % nlayer
																								snake_case_ :Any						=			vnp.copy()  # same in embedded
																								snake_case_ :List[Any]						=			torch.tensor(_lowercase	)
																								if key_name.startswith("""model/wte"""	):
																														snake_case_ :Tuple						=			"""lm_head.weight"""
																														snake_case_ :List[str]						=			vnp.copy()  # same in embedded
																														snake_case_ :List[Any]						=			torch.tensor(_lowercase	)
																		elif key_name.startswith("""model/wob"""	):
																								snake_case_ :str						=			"""final_logits_bias"""
																								snake_case_ :Any						=			vnp.copy()  # same in embedded
																								snake_case_ :List[Any]						=			state.reshape((1, -1)	)
																								snake_case_ :Union[str, Any]						=			torch.tensor(_lowercase	)
																		elif key_name == "model/dense/kernel":
																								snake_case_ :str						=			"""model.last_project.weight"""
																								snake_case_ :Dict						=			vnp.transpose([1, 0]	).copy()  # Mesh-Tensorflow is a diagonal matrix
																								snake_case_ :int						=			torch.tensor(_lowercase	)
																		elif key_name == "model/dense_1/bias":
																								snake_case_ :Optional[int]						=			"""model.last_project.bias"""
																								snake_case_ :Tuple						=			vnp.copy()  # same because it is one dimensional
																								snake_case_ :Any						=			torch.tensor(_lowercase	)
						torch.save(_lowercase,					args.output	)


if __name__ == "__main__":
						__a      =				argparse.ArgumentParser(
						    description="model converter.", formatter_class=argparse.ArgumentDefaultsHelpFormatter
						)
						parser.add_argument("--tf_model_dir", metavar="PATH", type=str, required=True, help="import model")
						parser.add_argument("--output", metavar="PATH", type=str, required=True, help="output model")
						__a      =				parser.parse_args()
						convert_tf_gptsan_to_pt(args)
