



import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
	from transformers import LlamaTokenizerFast
except ImportError as e:
	warnings.warn(e)
	warnings.warn(
	    'The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'
	)
	a	:   Optional[int]       			=       None
a	:   Optional[Any]       			=       {
    '7B': 11_008,
    '13B': 13_824,
    '30B': 17_920,
    '65B': 22_016,
    '70B': 28_672,
}
a	:   List[str]       			=       {
    '7B': 1,
    '7Bf': 1,
    '13B': 2,
    '13Bf': 2,
    '30B': 4,
    '65B': 8,
    '70B': 8,
    '70Bf': 8,
}







def       lowerCAmelCase_     (lowerCAmelCase__:						List[str]						,		lowerCAmelCase__:						Optional[int]=1						,		lowerCAmelCase__:						Dict=2_5_6   ):





			"""simple docstring"""


			return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3   )   ) + multiple_of - 1) // multiple_of)







def       lowerCAmelCase_     (lowerCAmelCase__:						List[str]   ):





			"""simple docstring"""


			with open(_snake_case						,		"""r"""   ) as f:
						return json.load(_snake_case   )







def       lowerCAmelCase_     (lowerCAmelCase__:						List[str]						,		lowerCAmelCase__:						int   ):





			"""simple docstring"""


			with open(_snake_case						,		"""w"""   ) as f:
						json.dump(_snake_case						,		_snake_case   )







def       lowerCAmelCase_     (lowerCAmelCase__:						Optional[Any]						,		lowerCAmelCase__:						Tuple						,		lowerCAmelCase__:						Any						,		lowerCAmelCase__:						Union[str, Any]=True   ):





			"""simple docstring"""


			os.makedirs(_snake_case						,		exist_ok=_snake_case   )
			UpperCAmelCase_:						Tuple					=			os.path.join(_snake_case						,		"""tmp"""   )
			os.makedirs(_snake_case						,		exist_ok=_snake_case   )

			UpperCAmelCase_:						Any					=			read_json(os.path.join(_snake_case						,		"""params.json"""   )   )
			UpperCAmelCase_:						Union[str, Any]					=			NUM_SHARDS[model_size]
			UpperCAmelCase_:						Optional[int]					=			params["""n_layers"""]
			UpperCAmelCase_:						str					=			params["""n_heads"""]
			UpperCAmelCase_:						List[str]					=			n_heads // num_shards
			UpperCAmelCase_:						List[str]					=			params["""dim"""]
			UpperCAmelCase_:						Optional[Any]					=			dim // n_heads
			UpperCAmelCase_:						str					=			10000.0
			UpperCAmelCase_:						List[str]					=			1.0 / (base ** (torch.arange(0						,		_snake_case						,		2   ).float() / dims_per_head))

			if "n_kv_heads" in params:
						UpperCAmelCase_:						List[str]					=			params["""n_kv_heads"""]  # for GQA / MQA
						UpperCAmelCase_:						str					=			n_heads_per_shard // num_key_value_heads
						UpperCAmelCase_:						str					=			dim // num_key_value_heads
			else:  # compatibility with other checkpoints
						UpperCAmelCase_:						List[Any]					=			n_heads
						UpperCAmelCase_:						List[Any]					=			n_heads_per_shard
						UpperCAmelCase_:						List[Any]					=			dim

			# permute for sliced rotary
			def permute(lowerCAmelCase__:						Union[str, Any]						,		lowerCAmelCase__:						List[Any]=n_heads						,		lowerCAmelCase__:						str=dim						,		lowerCAmelCase__:						int=dim   ):
						return w.view(_snake_case						,		dima // n_heads // 2						,		2						,		_snake_case   ).transpose(1						,		2   ).reshape(_snake_case						,		_snake_case   )

			print(F'Fetching all parameters from the checkpoint at {input_base_path}.'   )
			# Load weights
			if model_size == "7B":
						# Not sharded
						# (The sharded implementation would also work, but this is simpler.)
						UpperCAmelCase_:						List[Any]					=			torch.load(os.path.join(_snake_case						,		"""consolidated.00.pth"""   )						,		map_location="""cpu"""   )
			else:
						# Sharded
						UpperCAmelCase_:						Tuple					=			[
						    torch.load(os.path.join(_snake_case						,		F'consolidated.{i:02d}.pth'   )						,		map_location="""cpu"""   )
						    for i in range(_snake_case   )
						]
			UpperCAmelCase_:						List[Any]					=			0
			UpperCAmelCase_:						List[Any]					=			{"""weight_map""": {}}
			for layer_i in range(_snake_case   ):
						UpperCAmelCase_:						Optional[int]					=			F'pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'
						if model_size == "7B":
									# Unsharded
									UpperCAmelCase_:						Dict					=			{
									    F'model.layers.{layer_i}.self_attn.q_proj.weight': permute(
									        loaded[F'layers.{layer_i}.attention.wq.weight']   ),
									    F'model.layers.{layer_i}.self_attn.k_proj.weight': permute(
									        loaded[F'layers.{layer_i}.attention.wk.weight']   ),
									    F'model.layers.{layer_i}.self_attn.v_proj.weight': loaded[F'layers.{layer_i}.attention.wv.weight'],
									    F'model.layers.{layer_i}.self_attn.o_proj.weight': loaded[F'layers.{layer_i}.attention.wo.weight'],
									    F'model.layers.{layer_i}.mlp.gate_proj.weight': loaded[F'layers.{layer_i}.feed_forward.w1.weight'],
									    F'model.layers.{layer_i}.mlp.down_proj.weight': loaded[F'layers.{layer_i}.feed_forward.w2.weight'],
									    F'model.layers.{layer_i}.mlp.up_proj.weight': loaded[F'layers.{layer_i}.feed_forward.w3.weight'],
									    F'model.layers.{layer_i}.input_layernorm.weight': loaded[F'layers.{layer_i}.attention_norm.weight'],
									    F'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[F'layers.{layer_i}.ffn_norm.weight'],
									}
						else:
									# Sharded
									# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
									# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
									# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

									UpperCAmelCase_:						int					=			{
									    F'model.layers.{layer_i}.input_layernorm.weight': loaded[0][
									        F'layers.{layer_i}.attention_norm.weight'
									    ].clone(),
									    F'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[0][
									        F'layers.{layer_i}.ffn_norm.weight'
									    ].clone(),
									}
									UpperCAmelCase_:						int					=			permute(
									    torch.cat(
									        [
									            loaded[i][F'layers.{layer_i}.attention.wq.weight'].view(_snake_case						,		_snake_case						,		_snake_case   )
									            for i in range(_snake_case   )
									        ]						,		dim=0						,		).reshape(_snake_case						,		_snake_case   )   )
									UpperCAmelCase_:						List[Any]					=			permute(
									    torch.cat(
									        [
									            loaded[i][F'layers.{layer_i}.attention.wk.weight'].view(
									                _snake_case						,		_snake_case						,		_snake_case   )
									            for i in range(_snake_case   )
									        ]						,		dim=0						,		).reshape(_snake_case						,		_snake_case   )						,		_snake_case						,		_snake_case						,		_snake_case						,		)
									UpperCAmelCase_:						Dict					=			torch.cat(
									    [
									        loaded[i][F'layers.{layer_i}.attention.wv.weight'].view(
									            _snake_case						,		_snake_case						,		_snake_case   )
									        for i in range(_snake_case   )
									    ]						,		dim=0						,		).reshape(_snake_case						,		_snake_case   )

									UpperCAmelCase_:						Optional[Any]					=			torch.cat(
									    [loaded[i][F'layers.{layer_i}.attention.wo.weight'] for i in range(_snake_case   )]						,		dim=1   )
									UpperCAmelCase_:						Optional[int]					=			torch.cat(
									    [loaded[i][F'layers.{layer_i}.feed_forward.w1.weight'] for i in range(_snake_case   )]						,		dim=0   )
									UpperCAmelCase_:						Dict					=			torch.cat(
									    [loaded[i][F'layers.{layer_i}.feed_forward.w2.weight'] for i in range(_snake_case   )]						,		dim=1   )
									UpperCAmelCase_:						Any					=			torch.cat(
									    [loaded[i][F'layers.{layer_i}.feed_forward.w3.weight'] for i in range(_snake_case   )]						,		dim=0   )

						UpperCAmelCase_:						Optional[Any]					=			inv_freq
						for k, v in state_dict.items():
									UpperCAmelCase_:						Optional[int]					=			filename
									param_count += v.numel()
						torch.save(_snake_case						,		os.path.join(_snake_case						,		_snake_case   )   )

			UpperCAmelCase_:						int					=			F'pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'
			if model_size == "7B":
						# Unsharded
						UpperCAmelCase_:						Dict					=			{
						    """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
						    """model.norm.weight""": loaded["""norm.weight"""],
						    """lm_head.weight""": loaded["""output.weight"""],
						}
			else:
						UpperCAmelCase_:						Optional[Any]					=			{
						    """model.norm.weight""": loaded[0]["""norm.weight"""],
						    """model.embed_tokens.weight""": torch.cat(
						        [loaded[i]["""tok_embeddings.weight"""] for i in range(_snake_case   )]						,		dim=1   ),
						    """lm_head.weight""": torch.cat([loaded[i]["""output.weight"""] for i in range(_snake_case   )]						,		dim=0   ),
						}

			for k, v in state_dict.items():
						UpperCAmelCase_:						Tuple					=			filename
						param_count += v.numel()
			torch.save(_snake_case						,		os.path.join(_snake_case						,		_snake_case   )   )

			# Write configs
			UpperCAmelCase_:						str					=			{"""total_size""": param_count * 2}
			write_json(_snake_case						,		os.path.join(_snake_case						,		"""pytorch_model.bin.index.json"""   )   )
			UpperCAmelCase_:						Optional[int]					=			params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
			UpperCAmelCase_:						Tuple					=			params["""multiple_of"""] if """multiple_of""" in params else 2_5_6
			UpperCAmelCase_:						Union[str, Any]					=			LlamaConfig(
			    hidden_size=_snake_case						,		intermediate_size=compute_intermediate_size(_snake_case						,		_snake_case						,		_snake_case   )						,		num_attention_heads=params["""n_heads"""]						,		num_hidden_layers=params["""n_layers"""]						,		rms_norm_eps=params["""norm_eps"""]						,		num_key_value_heads=_snake_case						,		)
			config.save_pretrained(_snake_case   )

			# Make space so we can load the model properly now.
			del state_dict
			del loaded
			gc.collect()

			print("""Loading the checkpoint in a Llama model."""   )
			UpperCAmelCase_:						List[Any]					=			LlamaForCausalLM.from_pretrained(_snake_case						,		torch_dtype=torch.floataa						,		low_cpu_mem_usage=_snake_case   )
			# Avoid saving this as part of the config.
			del model.config._name_or_path

			print("""Saving in the Transformers format."""   )
			model.save_pretrained(_snake_case						,		safe_serialization=_snake_case   )
			shutil.rmtree(_snake_case   )







def       lowerCAmelCase_     (lowerCAmelCase__:						List[Any]						,		lowerCAmelCase__:						Optional[Any]   ):





			"""simple docstring"""


			UpperCAmelCase_:						int					=			LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
			print(F'Saving a {tokenizer_class.__name__} to {tokenizer_path}.'   )
			UpperCAmelCase_:						Tuple					=			tokenizer_class(_snake_case   )
			tokenizer.save_pretrained(_snake_case   )







def       lowerCAmelCase_     ():





			"""simple docstring"""


			UpperCAmelCase_:						int					=			argparse.ArgumentParser()
			parser.add_argument(
			    """--input_dir"""						,		help="""Location of LLaMA weights, which contains tokenizer.model and model folders"""						,		)
			parser.add_argument(
			    """--model_size"""						,		choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""]						,		)
			parser.add_argument(
			    """--output_dir"""						,		help="""Location to write HF model and tokenizer"""						,		)
			parser.add_argument("""--safe_serialization"""						,		type=_snake_case						,		help="""Whether or not to save using `safetensors`."""   )
			UpperCAmelCase_:						Optional[int]					=			parser.parse_args()
			if args.model_size != "tokenizer_only":
						write_model(
						    model_path=args.output_dir						,		input_base_path=os.path.join(args.input_dir						,		args.model_size   )						,		model_size=args.model_size						,		safe_serialization=args.safe_serialization						,		)
			UpperCAmelCase_:						Any					=			os.path.join(args.input_dir						,		"""tokenizer.model"""   )
			write_tokenizer(args.output_dir						,		_snake_case   )


if __name__ == "__main__":
	main()








"""simple docstring"""




def lowercase_  (   _snake_case   ,_snake_case						):

							if not (isinstance(_snake_case   ,_snake_case						) and isinstance(_snake_case   ,_snake_case						)):
														raise ValueError("""longest_common_substring() takes two strings for inputs"""						)

							SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   len(_snake_case						)
							SCREAMING_SNAKE_CASE__						:       int           =   len(_snake_case						)

							SCREAMING_SNAKE_CASE__						:       Dict           =   [[0] * (texta_length + 1) for _ in range(texta_length + 1						)]
							SCREAMING_SNAKE_CASE__						:       List[Any]           =   0
							SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   0

							for i in range(1   ,texta_length + 1						):
														for j in range(1   ,texta_length + 1						):
																					if texta[i - 1] == texta[j - 1]:
																												SCREAMING_SNAKE_CASE__						:       int           =   1 + dp[i - 1][j - 1]
																												if dp[i][j] > ans_length:
																																			SCREAMING_SNAKE_CASE__						:       List[Any]           =   i
																																			SCREAMING_SNAKE_CASE__						:       List[str]           =   dp[i][j]

							return texta[ans_index - ans_length : ans_index]


if __name__ == "__main__":
					import doctest

					doctest.testmod()




