


"""simple docstring"""






import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig


def __lowercase  (   lowerCamelCase_	:   Union[str, Any]    ,       lowerCamelCase_	:   int    ,       lowerCamelCase_	:   List[str]=0	):
				# Format the message.
				if name is None:
								SCREAMING_SNAKE_CASE__  				=					None
				else:
								SCREAMING_SNAKE_CASE__  				=					"." * max(0    ,       spaces - 2	) + "# {:" + str(50 - spaces	) + "s}"
								SCREAMING_SNAKE_CASE__  				=					fmt.format(lowerCamelCase_	)

				# Print and recurse (if needed).
				if isinstance(lowerCamelCase_    ,       lowerCamelCase_	):
								if msg is not None:
												print(lowerCamelCase_	)
								for k in val.keys():
												recursive_print(lowerCamelCase_    ,       val[k]    ,       spaces + 2	)
				elif isinstance(lowerCamelCase_    ,       torch.Tensor	):
								print(lowerCamelCase_    ,       ":"    ,       val.size()	)
				else:
								print(lowerCamelCase_    ,       ":"    ,       lowerCamelCase_	)


def __lowercase  (   lowerCamelCase_	:   int    ,       lowerCamelCase_	:   Dict    ,       lowerCamelCase_	:   Any    ,       lowerCamelCase_	:   Optional[int]	):
				# Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
				# for compatibility with later versions of NVIDIA Megatron-LM.
				# The inverse operation is performed inside Megatron-LM to read checkpoints:
				# https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
				# If param is the weight tensor of the self-attention block, the returned tensor
				# will have to be transposed one more time to be read by HuggingFace GPT2.
				SCREAMING_SNAKE_CASE__  				=					param.size()
				# other versions store [num_heads * num_splits * hidden_size, :]
				SCREAMING_SNAKE_CASE__  				=					(num_heads, num_splits, hidden_size) + input_shape[1:]
				SCREAMING_SNAKE_CASE__  				=					param.view(*lowerCamelCase_	)
				SCREAMING_SNAKE_CASE__  				=					param.transpose(0    ,       1	).contiguous()
				SCREAMING_SNAKE_CASE__  				=					param.view(*lowerCamelCase_	)
				return param


def __lowercase  (   lowerCamelCase_	:   int    ,       lowerCamelCase_	:   str	):
				SCREAMING_SNAKE_CASE__  				=					config.n_positions
				SCREAMING_SNAKE_CASE__  				=					config.n_layer
				SCREAMING_SNAKE_CASE__  				=					config.vocab_size
				SCREAMING_SNAKE_CASE__  				=					config.n_head
				SCREAMING_SNAKE_CASE__  				=					config.n_embd // config.n_head

				SCREAMING_SNAKE_CASE__  				=					sd_megatron["model.language_model.embedding.word_embeddings.weight"][:vocab_size, :]
				SCREAMING_SNAKE_CASE__  				=					{
				    "transformer.wte.weight": word_embeddings,
				    "transformer.wpe.weight": sd_megatron["model.language_model.embedding.position_embeddings.weight"],
				    "transformer.ln_f.weight": sd_megatron["model.language_model.encoder.final_layernorm.weight"],
				    "transformer.ln_f.bias": sd_megatron["model.language_model.encoder.final_layernorm.bias"],
				}

				SCREAMING_SNAKE_CASE__  				=					"model.language_model.encoder.layers."
				for i in range(lowerCamelCase_	):
								SCREAMING_SNAKE_CASE__  				=					torch.tril(torch.ones((n_positions, n_positions)    ,       dtype=torch.bool	)	)
								SCREAMING_SNAKE_CASE__  				=					causal_mask.view(1    ,       1    ,       lowerCamelCase_    ,       lowerCamelCase_	)
								SCREAMING_SNAKE_CASE__  				=					causal_mask
								SCREAMING_SNAKE_CASE__  				=					torch.tensor(-1e4    ,       dtype=torch.bfloataa	)

								SCREAMING_SNAKE_CASE__  				=					sd_megatron[F'''{pf}{i}.input_layernorm.weight''']
								SCREAMING_SNAKE_CASE__  				=					sd_megatron[F'''{pf}{i}.input_layernorm.bias''']

								SCREAMING_SNAKE_CASE__  				=					sd_megatron[F'''{pf}{i}.self_attention.query_key_value.weight''']
								SCREAMING_SNAKE_CASE__  				=					fix_query_key_value_ordering(lowerCamelCase_    ,       3    ,       lowerCamelCase_    ,       lowerCamelCase_	)
								SCREAMING_SNAKE_CASE__  				=					vala.transpose(0    ,       1	).contiguous()

								SCREAMING_SNAKE_CASE__  				=					sd_megatron[F'''{pf}{i}.self_attention.query_key_value.bias''']
								SCREAMING_SNAKE_CASE__  				=					fix_query_key_value_ordering(lowerCamelCase_    ,       3    ,       lowerCamelCase_    ,       lowerCamelCase_	)
								SCREAMING_SNAKE_CASE__  				=					vala

								SCREAMING_SNAKE_CASE__  				=					sd_megatron[F'''{pf}{i}.self_attention.dense.weight'''].transpose(
								    0    ,       1	)
								SCREAMING_SNAKE_CASE__  				=					sd_megatron[F'''{pf}{i}.self_attention.dense.bias''']
								SCREAMING_SNAKE_CASE__  				=					sd_megatron[F'''{pf}{i}.post_attention_layernorm.weight''']
								SCREAMING_SNAKE_CASE__  				=					sd_megatron[F'''{pf}{i}.post_attention_layernorm.bias''']
								SCREAMING_SNAKE_CASE__  				=					sd_megatron[F'''{pf}{i}.mlp.dense_h_to_4h.weight'''].transpose(0    ,       1	)
								SCREAMING_SNAKE_CASE__  				=					sd_megatron[F'''{pf}{i}.mlp.dense_h_to_4h.bias''']
								SCREAMING_SNAKE_CASE__  				=					sd_megatron[F'''{pf}{i}.mlp.dense_4h_to_h.weight'''].transpose(
								    0    ,       1	)
								SCREAMING_SNAKE_CASE__  				=					sd_megatron[F'''{pf}{i}.mlp.dense_4h_to_h.bias''']

				# For LM head, transformers' wants the matrix to weight embeddings.
				SCREAMING_SNAKE_CASE__  				=					word_embeddings

				return sd_hf


def __lowercase  (   lowerCamelCase_	:   str    ,       lowerCamelCase_	:   List[str]	):
				SCREAMING_SNAKE_CASE__  				=					64000
				SCREAMING_SNAKE_CASE__  				=					config_megatron["encoder_seq_length"]
				SCREAMING_SNAKE_CASE__  				=					config_megatron["hidden_size"]
				SCREAMING_SNAKE_CASE__  				=					config_megatron["num_layers"]
				SCREAMING_SNAKE_CASE__  				=					config_megatron["num_attention_heads"]
				SCREAMING_SNAKE_CASE__  				=					config_megatron["ffn_hidden_size"]
				SCREAMING_SNAKE_CASE__  				=					"gelu"
				SCREAMING_SNAKE_CASE__  				=					0.1
				SCREAMING_SNAKE_CASE__  				=					0.1
				SCREAMING_SNAKE_CASE__  				=					0.1
				SCREAMING_SNAKE_CASE__  				=					config_megatron["layernorm_epsilon"]  # 1e-5
				SCREAMING_SNAKE_CASE__  				=					config_megatron["init_method_std"]  # 0.02
				SCREAMING_SNAKE_CASE__  				=					config_megatron["apply_query_key_layer_scaling"]  # True
				SCREAMING_SNAKE_CASE__  				=					True
				SCREAMING_SNAKE_CASE__  				=					True

				# This identifies the 6.7B (7B) model which uses a different tokenizer
				if config_megatron["hidden_size"] == 4096:
								SCREAMING_SNAKE_CASE__  				=					1  # <|endoftext|>
								SCREAMING_SNAKE_CASE__  				=					1  # <|endoftext|>
								SCREAMING_SNAKE_CASE__  				=					0  # <unk>
				else:
								SCREAMING_SNAKE_CASE__  				=					2  # <s>
								SCREAMING_SNAKE_CASE__  				=					3  # <|endoftext|>
								SCREAMING_SNAKE_CASE__  				=					0  # <pad>

				return config_hf


def __lowercase  (   lowerCamelCase_	:   List[str]	):
				print(lowerCamelCase_	)

				SCREAMING_SNAKE_CASE__  				=					args.checkpoint_path
				SCREAMING_SNAKE_CASE__  				=					args.save_path
				if isfile(lowerCamelCase_	):
								raise FileNotFoundError(F'''ERROR! could not find file {checkpoint_path}'''	)

				# Load the model.
				SCREAMING_SNAKE_CASE__  				=					torch.load(lowerCamelCase_    ,       map_location="cpu"	)

				# Load the config.
				SCREAMING_SNAKE_CASE__  				=					checkpoint["hyper_parameters"]["cfg"]
				SCREAMING_SNAKE_CASE__  				=					GPTaConfig()
				SCREAMING_SNAKE_CASE__  				=					copy_config(config_hf=lowerCamelCase_    ,       config_megatron=lowerCamelCase_	)
				SCREAMING_SNAKE_CASE__  				=					["GPT2LMHeadModel"]

				SCREAMING_SNAKE_CASE__  				=					checkpoint["state_dict"]

				# Convert.
				print("Converting"	)
				SCREAMING_SNAKE_CASE__  				=					convert_megatron_checkpoint(lowerCamelCase_    ,       lowerCamelCase_	)

				# Print the structure of converted state dict.
				if args.print_checkpoint_structure:
								recursive_print(lowerCamelCase_    ,       lowerCamelCase_	)

				SCREAMING_SNAKE_CASE__  				=					"GPTSw3Tokenizer"

				# Store the config to file.
				print("Saving config"	)
				config_hf.save_pretrained(lowerCamelCase_	)

				# Store the state_dict to file.
				SCREAMING_SNAKE_CASE__  				=					os.path.join(lowerCamelCase_    ,       "pytorch_model.bin"	)
				print(F'''Saving checkpoint to "{output_checkpoint_file}"'''	)
				torch.save(lowerCamelCase_    ,       lowerCamelCase_	)


if __name__ == "__main__":
	_lowerCamelCase  =	argparse.ArgumentParser()
	parser.add_argument(
	    '--checkpoint_path',
	    type=str,
	    required=True,
	    help='e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000',
	)
	parser.add_argument('--save_path', type=str, required=True, help='e.g. /home/user/gpt-sw3/hf')
	parser.add_argument('--print-checkpoint-structure', action='store_true')
	_lowerCamelCase  =	parser.parse_args()
	main(_args)




"""simple docstring"""






from __future__ import annotations

from collections.abc import Generator


def __lowercase  (   lowerCamelCase_	:   int	):
				if not isinstance(lowerCamelCase_    ,       lowerCamelCase_	) or n < 1:
								raise Exception("Sequence only defined for positive integers"	)

				yield n
				while n != 1:
								if n % 2 == 0:
												n //= 2
								else:
												SCREAMING_SNAKE_CASE__  				=					3 * n + 1
								yield n


def __lowercase  (   ):
				SCREAMING_SNAKE_CASE__  				=					43
				SCREAMING_SNAKE_CASE__  				=					tuple(collatz_sequence(lowerCamelCase_	)	)
				print(lowerCamelCase_	)
				print(F'''Collatz sequence from {n} took {len(lowerCamelCase_	)} steps.'''	)


if __name__ == "__main__":
	main()
