



'''simple docstring'''




import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig



def  __lowerCamelCase    (   _lowercase      ,							_lowercase      ,							_lowercase=0						) ->     Any:
							# Format the message.
							if name is None:
														UpperCAmelCase	:						int					=						None
							else:
														UpperCAmelCase	:						Dict					=						""".""" * max(0      ,							spaces - 2						) + """# {:""" + str(5_0 - spaces						) + """s}"""
														UpperCAmelCase	:						int					=						fmt.format(_lowercase						)

							# Print and recurse (if needed).
							if isinstance(_lowercase      ,							_lowercase						):
														if msg is not None:
																					print(_lowercase						)
														for k in val.keys():
																					recursive_print(_lowercase      ,							val[k]      ,							spaces + 2						)
							elif isinstance(_lowercase      ,							torch.Tensor						):
														print(_lowercase      ,							""":"""      ,							val.size()						)
							else:
														print(_lowercase      ,							""":"""      ,							_lowercase						)



def  __lowerCamelCase    (   _lowercase      ,							_lowercase      ,							_lowercase      ,							_lowercase						) ->     int:
							# Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
							# for compatibility with later versions of NVIDIA Megatron-LM.
							# The inverse operation is performed inside Megatron-LM to read checkpoints:
							# https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
							# If param is the weight tensor of the self-attention block, the returned tensor
							# will have to be transposed one more time to be read by HuggingFace GPT2.
							UpperCAmelCase	:						Tuple					=						param.size()
							# other versions store [num_heads * num_splits * hidden_size, :]
							UpperCAmelCase	:						Optional[int]					=						(num_heads, num_splits, hidden_size) + input_shape[1:]
							UpperCAmelCase	:						int					=						param.view(*_lowercase						)
							UpperCAmelCase	:						Dict					=						param.transpose(0      ,							1						).contiguous()
							UpperCAmelCase	:						Any					=						param.view(*_lowercase						)
							return param



def  __lowerCamelCase    (   _lowercase      ,							_lowercase						) ->     Any:
							UpperCAmelCase	:						int					=						config.n_positions
							UpperCAmelCase	:						str					=						config.n_layer
							UpperCAmelCase	:						Any					=						config.vocab_size
							UpperCAmelCase	:						Union[str, Any]					=						config.n_head
							UpperCAmelCase	:						Optional[Any]					=						config.n_embd // config.n_head

							UpperCAmelCase	:						int					=						sd_megatron["""model.language_model.embedding.word_embeddings.weight"""][:vocab_size, :]
							UpperCAmelCase	:						Optional[int]					=						{
							    """transformer.wte.weight""": word_embeddings,
							    """transformer.wpe.weight""": sd_megatron["""model.language_model.embedding.position_embeddings.weight"""],
							    """transformer.ln_f.weight""": sd_megatron["""model.language_model.encoder.final_layernorm.weight"""],
							    """transformer.ln_f.bias""": sd_megatron["""model.language_model.encoder.final_layernorm.bias"""],
							}

							UpperCAmelCase	:						int					=						"""model.language_model.encoder.layers."""
							for i in range(_lowercase						):
														UpperCAmelCase	:						List[Any]					=						torch.tril(torch.ones((n_positions, n_positions)      ,							dtype=torch.bool						)						)
														UpperCAmelCase	:						List[Any]					=						causal_mask.view(1      ,							1      ,							_lowercase      ,							_lowercase						)
														UpperCAmelCase	:						str					=						causal_mask
														UpperCAmelCase	:						int					=						torch.tensor(-1e4      ,							dtype=torch.bfloataa						)

														UpperCAmelCase	:						List[str]					=						sd_megatron[F'''{pf}{i}.input_layernorm.weight''']
														UpperCAmelCase	:						int					=						sd_megatron[F'''{pf}{i}.input_layernorm.bias''']

														UpperCAmelCase	:						int					=						sd_megatron[F'''{pf}{i}.self_attention.query_key_value.weight''']
														UpperCAmelCase	:						Tuple					=						fix_query_key_value_ordering(_lowercase      ,							3      ,							_lowercase      ,							_lowercase						)
														UpperCAmelCase	:						Any					=						vala.transpose(0      ,							1						).contiguous()

														UpperCAmelCase	:						Optional[int]					=						sd_megatron[F'''{pf}{i}.self_attention.query_key_value.bias''']
														UpperCAmelCase	:						Any					=						fix_query_key_value_ordering(_lowercase      ,							3      ,							_lowercase      ,							_lowercase						)
														UpperCAmelCase	:						List[Any]					=						vala

														UpperCAmelCase	:						Optional[Any]					=						sd_megatron[F'''{pf}{i}.self_attention.dense.weight'''].transpose(
														    0      ,							1						)
														UpperCAmelCase	:						Optional[int]					=						sd_megatron[F'''{pf}{i}.self_attention.dense.bias''']
														UpperCAmelCase	:						Tuple					=						sd_megatron[F'''{pf}{i}.post_attention_layernorm.weight''']
														UpperCAmelCase	:						Dict					=						sd_megatron[F'''{pf}{i}.post_attention_layernorm.bias''']
														UpperCAmelCase	:						str					=						sd_megatron[F'''{pf}{i}.mlp.dense_h_to_4h.weight'''].transpose(0      ,							1						)
														UpperCAmelCase	:						int					=						sd_megatron[F'''{pf}{i}.mlp.dense_h_to_4h.bias''']
														UpperCAmelCase	:						str					=						sd_megatron[F'''{pf}{i}.mlp.dense_4h_to_h.weight'''].transpose(
														    0      ,							1						)
														UpperCAmelCase	:						int					=						sd_megatron[F'''{pf}{i}.mlp.dense_4h_to_h.bias''']

							# For LM head, transformers' wants the matrix to weight embeddings.
							UpperCAmelCase	:						Optional[int]					=						word_embeddings

							return sd_hf



def  __lowerCamelCase    (   _lowercase      ,							_lowercase						) ->     Any:
							UpperCAmelCase	:						str					=						6_4_0_0_0
							UpperCAmelCase	:						List[str]					=						config_megatron["""encoder_seq_length"""]
							UpperCAmelCase	:						Optional[int]					=						config_megatron["""hidden_size"""]
							UpperCAmelCase	:						List[Any]					=						config_megatron["""num_layers"""]
							UpperCAmelCase	:						List[Any]					=						config_megatron["""num_attention_heads"""]
							UpperCAmelCase	:						Dict					=						config_megatron["""ffn_hidden_size"""]
							UpperCAmelCase	:						str					=						"""gelu"""
							UpperCAmelCase	:						Optional[int]					=						0.1
							UpperCAmelCase	:						List[str]					=						0.1
							UpperCAmelCase	:						Tuple					=						0.1
							UpperCAmelCase	:						Optional[int]					=						config_megatron["""layernorm_epsilon"""]  # 1e-5
							UpperCAmelCase	:						Optional[Any]					=						config_megatron["""init_method_std"""]  # 0.02
							UpperCAmelCase	:						Optional[Any]					=						config_megatron["""apply_query_key_layer_scaling"""]  # True
							UpperCAmelCase	:						str					=						True
							UpperCAmelCase	:						Tuple					=						True

							# This identifies the 6.7B (7B) model which uses a different tokenizer
							if config_megatron["hidden_size"] == 4_0_9_6:
														UpperCAmelCase	:						Optional[Any]					=						1  # <|endoftext|>
														UpperCAmelCase	:						int					=						1  # <|endoftext|>
														UpperCAmelCase	:						Optional[Any]					=						0  # <unk>
							else:
														UpperCAmelCase	:						Dict					=						2  # <s>
														UpperCAmelCase	:						Optional[Any]					=						3  # <|endoftext|>
														UpperCAmelCase	:						List[str]					=						0  # <pad>

							return config_hf



def  __lowerCamelCase    (   _lowercase						) ->     Union[str, Any]:
							print(_lowercase						)

							UpperCAmelCase	:						Tuple					=						args.checkpoint_path
							UpperCAmelCase	:						List[Any]					=						args.save_path
							if isfile(_lowercase						):
														raise FileNotFoundError(F'''ERROR! could not find file {checkpoint_path}'''						)

							# Load the model.
							UpperCAmelCase	:						List[Any]					=						torch.load(_lowercase      ,							map_location="""cpu"""						)

							# Load the config.
							UpperCAmelCase	:						str					=						checkpoint["""hyper_parameters"""]["""cfg"""]
							UpperCAmelCase	:						Optional[Any]					=						GPTaConfig()
							UpperCAmelCase	:						str					=						copy_config(config_hf=_lowercase      ,							config_megatron=_lowercase						)
							UpperCAmelCase	:						str					=						["""GPT2LMHeadModel"""]

							UpperCAmelCase	:						List[Any]					=						checkpoint["""state_dict"""]

							# Convert.
							print("""Converting"""						)
							UpperCAmelCase	:						Dict					=						convert_megatron_checkpoint(_lowercase      ,							_lowercase						)

							# Print the structure of converted state dict.
							if args.print_checkpoint_structure:
														recursive_print(_lowercase      ,							_lowercase						)

							UpperCAmelCase	:						Tuple					=						"""GPTSw3Tokenizer"""

							# Store the config to file.
							print("""Saving config"""						)
							config_hf.save_pretrained(_lowercase						)

							# Store the state_dict to file.
							UpperCAmelCase	:						Tuple					=						os.path.join(_lowercase      ,							"""pytorch_model.bin"""						)
							print(F'''Saving checkpoint to "{output_checkpoint_file}"'''						)
							torch.save(_lowercase      ,							_lowercase						)


if __name__ == "__main__":
							a  :     Dict       =       argparse.ArgumentParser()
							parser.add_argument(
							    """--checkpoint_path""",
							    type=str,
							    required=True,
							    help="""e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000""",
							)
							parser.add_argument("""--save_path""", type=str, required=True, help="""e.g. /home/user/gpt-sw3/hf""")
							parser.add_argument("""--print-checkpoint-structure""", action="""store_true""")
							a  :     Optional[Any]       =       parser.parse_args()
							main(_args)






'''simple docstring'''




import sys



def  __lowerCamelCase    (   _lowercase      ,							_lowercase						) ->     int:
							if not numbers:
														return 0
							if target == 0 and target in numbers:
														return 0
							if not isinstance(_lowercase      ,							(list, tuple)						) or not all(
							    isinstance(_lowercase      ,							_lowercase						) for number in numbers						):
														raise ValueError("""numbers must be an iterable of integers"""						)

							UpperCAmelCase	:						Optional[Any]					=						0
							UpperCAmelCase	:						List[str]					=						sys.maxsize

							while right < len(_lowercase						):
														curr_sum += numbers[right]
														while curr_sum >= target and left <= right:
																					UpperCAmelCase	:						Optional[Any]					=						min(_lowercase      ,							right - left + 1						)
																					curr_sum -= numbers[left]
																					left += 1
														right += 1

							return 0 if min_len == sys.maxsize else min_len

