






import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig

def   SCREAMING_SNAKE_CASE_				(				__magic_name__   :    Optional[Any]      ,     __magic_name__   :    int      ,     __magic_name__   :    List[Any]=0   )						->     Dict:






	"""simple docstring"""



	if name is None:
		UpperCamelCase	:str		=      None
	else:
		UpperCamelCase	:str		=      """.""" * max(0      ,     spaces - 2   ) + """# {:""" + str(50 - spaces   ) + """s}"""
		UpperCamelCase	:str		=      fmt.format(UpperCAmelCase_   )

	# Print and recurse (if needed).
	if isinstance(UpperCAmelCase_      ,     UpperCAmelCase_   ):
		if msg is not None:
			print(UpperCAmelCase_   )
		for k in val.keys():
			recursive_print(UpperCAmelCase_      ,     val[k]      ,     spaces + 2   )
	elif isinstance(UpperCAmelCase_      ,     torch.Tensor   ):
		print(UpperCAmelCase_      ,     """:"""      ,     val.size()   )
	else:
		print(UpperCAmelCase_      ,     """:"""      ,     UpperCAmelCase_   )

def   SCREAMING_SNAKE_CASE_				(				__magic_name__   :    Optional[int]      ,     __magic_name__   :    Dict      ,     __magic_name__   :    Optional[Any]      ,     __magic_name__   :    Union[str, Any]   )						->     str:






	"""simple docstring"""



	UpperCamelCase	:Any		=      param.size()
	# other versions store [num_heads * num_splits * hidden_size, :]
	UpperCamelCase	:int		=      (num_heads, num_splits, hidden_size) + input_shape[1:]
	UpperCamelCase	:List[str]		=      param.view(*UpperCAmelCase_   )
	UpperCamelCase	:Tuple		=      param.transpose(0      ,     1   ).contiguous()
	UpperCamelCase	:Optional[Any]		=      param.view(*UpperCAmelCase_   )
	return param

def   SCREAMING_SNAKE_CASE_				(				__magic_name__   :    Optional[Any]      ,     __magic_name__   :    Optional[Any]   )						->     str:






	"""simple docstring"""



	UpperCamelCase	:List[Any]		=      config.n_positions
	UpperCamelCase	:Union[str, Any]		=      config.n_layer
	UpperCamelCase	:List[str]		=      config.vocab_size
	UpperCamelCase	:List[Any]		=      config.n_head
	UpperCamelCase	:List[str]		=      config.n_embd // config.n_head

	UpperCamelCase	:List[Any]		=      sd_megatron["""model.language_model.embedding.word_embeddings.weight"""][:vocab_size, :]
	UpperCamelCase	:Tuple		=      {
	    """transformer.wte.weight""": word_embeddings,
	    """transformer.wpe.weight""": sd_megatron["""model.language_model.embedding.position_embeddings.weight"""],
	    """transformer.ln_f.weight""": sd_megatron["""model.language_model.encoder.final_layernorm.weight"""],
	    """transformer.ln_f.bias""": sd_megatron["""model.language_model.encoder.final_layernorm.bias"""],
	}

	UpperCamelCase	:Optional[int]		=      """model.language_model.encoder.layers."""
	for i in range(UpperCAmelCase_   ):
		UpperCamelCase	:List[Any]		=      torch.tril(torch.ones((n_positions, n_positions)      ,     dtype=torch.bool   )   )
		UpperCamelCase	:Dict		=      causal_mask.view(1      ,     1      ,     UpperCAmelCase_      ,     UpperCAmelCase_   )
		UpperCamelCase	:Optional[int]		=      causal_mask
		UpperCamelCase	:List[str]		=      torch.tensor(-1E4      ,     dtype=torch.bfloataa   )

		UpperCamelCase	:Union[str, Any]		=      sd_megatron[f"""{pf}{i}.input_layernorm.weight"""]
		UpperCamelCase	:Any		=      sd_megatron[f"""{pf}{i}.input_layernorm.bias"""]

		UpperCamelCase	:str		=      sd_megatron[f"""{pf}{i}.self_attention.query_key_value.weight"""]
		UpperCamelCase	:Optional[Any]		=      fix_query_key_value_ordering(UpperCAmelCase_      ,     3      ,     UpperCAmelCase_      ,     UpperCAmelCase_   )
		UpperCamelCase	:Optional[Any]		=      vala.transpose(0      ,     1   ).contiguous()

		UpperCamelCase	:Any		=      sd_megatron[f"""{pf}{i}.self_attention.query_key_value.bias"""]
		UpperCamelCase	:Union[str, Any]		=      fix_query_key_value_ordering(UpperCAmelCase_      ,     3      ,     UpperCAmelCase_      ,     UpperCAmelCase_   )
		UpperCamelCase	:Union[str, Any]		=      vala

		UpperCamelCase	:List[str]		=      sd_megatron[f"""{pf}{i}.self_attention.dense.weight"""].transpose(
		    0      ,     1   )
		UpperCamelCase	:List[str]		=      sd_megatron[f"""{pf}{i}.self_attention.dense.bias"""]
		UpperCamelCase	:int		=      sd_megatron[f"""{pf}{i}.post_attention_layernorm.weight"""]
		UpperCamelCase	:Tuple		=      sd_megatron[f"""{pf}{i}.post_attention_layernorm.bias"""]
		UpperCamelCase	:str		=      sd_megatron[f"""{pf}{i}.mlp.dense_h_to_4h.weight"""].transpose(0      ,     1   )
		UpperCamelCase	:List[str]		=      sd_megatron[f"""{pf}{i}.mlp.dense_h_to_4h.bias"""]
		UpperCamelCase	:List[Any]		=      sd_megatron[f"""{pf}{i}.mlp.dense_4h_to_h.weight"""].transpose(
		    0      ,     1   )
		UpperCamelCase	:List[str]		=      sd_megatron[f"""{pf}{i}.mlp.dense_4h_to_h.bias"""]

	# For LM head, transformers' wants the matrix to weight embeddings.
	UpperCamelCase	:Tuple		=      word_embeddings

	return sd_hf

def   SCREAMING_SNAKE_CASE_				(				__magic_name__   :    int      ,     __magic_name__   :    List[str]   )						->     Any:






	"""simple docstring"""



	UpperCamelCase	:Optional[int]		=      6_4000
	UpperCamelCase	:List[Any]		=      config_megatron["""encoder_seq_length"""]
	UpperCamelCase	:Dict		=      config_megatron["""hidden_size"""]
	UpperCamelCase	:Optional[int]		=      config_megatron["""num_layers"""]
	UpperCamelCase	:List[Any]		=      config_megatron["""num_attention_heads"""]
	UpperCamelCase	:Dict		=      config_megatron["""ffn_hidden_size"""]
	UpperCamelCase	:Optional[int]		=      """gelu"""
	UpperCamelCase	:Dict		=      0.1
	UpperCamelCase	:Optional[Any]		=      0.1
	UpperCamelCase	:List[str]		=      0.1
	UpperCamelCase	:List[str]		=      config_megatron["""layernorm_epsilon"""]  # 1e-5
	UpperCamelCase	:Optional[Any]		=      config_megatron["""init_method_std"""]  # 0.02
	UpperCamelCase	:List[Any]		=      config_megatron["""apply_query_key_layer_scaling"""]  # True
	UpperCamelCase	:List[str]		=      True
	UpperCamelCase	:List[Any]		=      True

	# This identifies the 6.7B (7B) model which uses a different tokenizer
	if config_megatron["hidden_size"] == 4096:
		UpperCamelCase	:Optional[Any]		=      1  # <|endoftext|>
		UpperCamelCase	:int		=      1  # <|endoftext|>
		UpperCamelCase	:List[Any]		=      0  # <unk>
	else:
		UpperCamelCase	:Any		=      2  # <s>
		UpperCamelCase	:int		=      3  # <|endoftext|>
		UpperCamelCase	:Tuple		=      0  # <pad>

	return config_hf

def   SCREAMING_SNAKE_CASE_				(				__magic_name__   :    int   )						->     Optional[int]:






	"""simple docstring"""



	print(UpperCAmelCase_   )

	UpperCamelCase	:List[str]		=      args.checkpoint_path
	UpperCamelCase	:Any		=      args.save_path
	if isfile(UpperCAmelCase_   ):
		raise FileNotFoundError(f"""ERROR! could not find file {checkpoint_path}"""   )

	# Load the model.
	UpperCamelCase	:Optional[Any]		=      torch.load(UpperCAmelCase_      ,     map_location="""cpu"""   )

	# Load the config.
	UpperCamelCase	:int		=      checkpoint["""hyper_parameters"""]["""cfg"""]
	UpperCamelCase	:Dict		=      GPTaConfig()
	UpperCamelCase	:Tuple		=      copy_config(config_hf=UpperCAmelCase_      ,     config_megatron=UpperCAmelCase_   )
	UpperCamelCase	:Tuple		=      ["""GPT2LMHeadModel"""]

	UpperCamelCase	:List[Any]		=      checkpoint["""state_dict"""]

	# Convert.
	print("""Converting"""   )
	UpperCamelCase	:str		=      convert_megatron_checkpoint(UpperCAmelCase_      ,     UpperCAmelCase_   )

	# Print the structure of converted state dict.
	if args.print_checkpoint_structure:
		recursive_print(UpperCAmelCase_      ,     UpperCAmelCase_   )

	UpperCamelCase	:Optional[int]		=      """GPTSw3Tokenizer"""

	# Store the config to file.
	print("""Saving config"""   )
	config_hf.save_pretrained(UpperCAmelCase_   )

	# Store the state_dict to file.
	UpperCamelCase	:Any		=      os.path.join(UpperCAmelCase_      ,     """pytorch_model.bin"""   )
	print(f"""Saving checkpoint to \"{output_checkpoint_file}\""""   )
	torch.save(UpperCAmelCase_      ,     UpperCAmelCase_   )


if __name__ == "__main__":
	UpperCAmelCase_					:		Dict           = argparse.ArgumentParser()
	parser.add_argument(
	    '''--checkpoint_path''',
	    type=str,
	    required=True,
	    help='''e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000''',
	)
	parser.add_argument('''--save_path''', type=str, required=True, help='''e.g. /home/user/gpt-sw3/hf''')
	parser.add_argument('''--print-checkpoint-structure''', action='''store_true''')
	UpperCAmelCase_					:		List[str]           = parser.parse_args()
	main(_args)






def 							__lowerCAmelCase			(UpperCAmelCase_		):
    return n * (2 * n - 1)
def 							__lowerCAmelCase			(UpperCAmelCase_		):
    a__  			=							(1 + 24 * n) ** 0.5
    return ((1 + root) / 6) % 1 == 0
def 							__lowerCAmelCase			(UpperCAmelCase_ = 144		):
    a__  			=							start
    a__  			=							hexagonal_num(UpperCAmelCase_		)
    while not is_pentagonal(UpperCAmelCase_		):
        n += 1
        a__  			=							hexagonal_num(UpperCAmelCase_		)
    return num


if __name__ == "__main__":
 print(F"{solution()} = ")






