




import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig



def    UpperCAmelCase							(  _snake_case    ,			_snake_case    ,			_snake_case=0				):
							# Format the message.
							if name is None:
														lowerCAmelCase =		None
							else:
														lowerCAmelCase =		'''.''' * max(0    ,			spaces - 2				) + '''# {:''' + str(50 - spaces				) + '''s}'''
														lowerCAmelCase =		fmt.format(_snake_case				)

							# Print and recurse (if needed).
							if isinstance(_snake_case    ,			_snake_case				):
														if msg is not None:
																					print(_snake_case				)
														for k in val.keys():
																					recursive_print(_snake_case    ,			val[k]    ,			spaces + 2				)
							elif isinstance(_snake_case    ,			torch.Tensor				):
														print(_snake_case    ,			''':'''    ,			val.size()				)
							else:
														print(_snake_case    ,			''':'''    ,			_snake_case				)



def    UpperCAmelCase							(  _snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case				):
							# Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
							# for compatibility with later versions of NVIDIA Megatron-LM.
							# The inverse operation is performed inside Megatron-LM to read checkpoints:
							# https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
							# If param is the weight tensor of the self-attention block, the returned tensor
							# will have to be transposed one more time to be read by HuggingFace GPT2.
							lowerCAmelCase =		param.size()
							# other versions store [num_heads * num_splits * hidden_size, :]
							lowerCAmelCase =		(num_heads, num_splits, hidden_size) + input_shape[1:]
							lowerCAmelCase =		param.view(*_snake_case				)
							lowerCAmelCase =		param.transpose(0    ,			1				).contiguous()
							lowerCAmelCase =		param.view(*_snake_case				)
							return param



def    UpperCAmelCase							(  _snake_case    ,			_snake_case				):
							lowerCAmelCase =		config.n_positions
							lowerCAmelCase =		config.n_layer
							lowerCAmelCase =		config.vocab_size
							lowerCAmelCase =		config.n_head
							lowerCAmelCase =		config.n_embd // config.n_head

							lowerCAmelCase =		sd_megatron['''model.language_model.embedding.word_embeddings.weight'''][:vocab_size, :]
							lowerCAmelCase =		{
							    '''transformer.wte.weight''': word_embeddings,
							    '''transformer.wpe.weight''': sd_megatron['''model.language_model.embedding.position_embeddings.weight'''],
							    '''transformer.ln_f.weight''': sd_megatron['''model.language_model.encoder.final_layernorm.weight'''],
							    '''transformer.ln_f.bias''': sd_megatron['''model.language_model.encoder.final_layernorm.bias'''],
							}

							lowerCAmelCase =		'''model.language_model.encoder.layers.'''
							for i in range(_snake_case				):
														lowerCAmelCase =		torch.tril(torch.ones((n_positions, n_positions)    ,			dtype=torch.bool				)				)
														lowerCAmelCase =		causal_mask.view(1    ,			1    ,			_snake_case    ,			_snake_case				)
														lowerCAmelCase =		causal_mask
														lowerCAmelCase =		torch.tensor(-1E4    ,			dtype=torch.bfloataa				)

														lowerCAmelCase =		sd_megatron[F"""{pf}{i}.input_layernorm.weight"""]
														lowerCAmelCase =		sd_megatron[F"""{pf}{i}.input_layernorm.bias"""]

														lowerCAmelCase =		sd_megatron[F"""{pf}{i}.self_attention.query_key_value.weight"""]
														lowerCAmelCase =		fix_query_key_value_ordering(_snake_case    ,			3    ,			_snake_case    ,			_snake_case				)
														lowerCAmelCase =		vala.transpose(0    ,			1				).contiguous()

														lowerCAmelCase =		sd_megatron[F"""{pf}{i}.self_attention.query_key_value.bias"""]
														lowerCAmelCase =		fix_query_key_value_ordering(_snake_case    ,			3    ,			_snake_case    ,			_snake_case				)
														lowerCAmelCase =		vala

														lowerCAmelCase =		sd_megatron[F"""{pf}{i}.self_attention.dense.weight"""].transpose(
														    0    ,			1				)
														lowerCAmelCase =		sd_megatron[F"""{pf}{i}.self_attention.dense.bias"""]
														lowerCAmelCase =		sd_megatron[F"""{pf}{i}.post_attention_layernorm.weight"""]
														lowerCAmelCase =		sd_megatron[F"""{pf}{i}.post_attention_layernorm.bias"""]
														lowerCAmelCase =		sd_megatron[F"""{pf}{i}.mlp.dense_h_to_4h.weight"""].transpose(0    ,			1				)
														lowerCAmelCase =		sd_megatron[F"""{pf}{i}.mlp.dense_h_to_4h.bias"""]
														lowerCAmelCase =		sd_megatron[F"""{pf}{i}.mlp.dense_4h_to_h.weight"""].transpose(
														    0    ,			1				)
														lowerCAmelCase =		sd_megatron[F"""{pf}{i}.mlp.dense_4h_to_h.bias"""]

							# For LM head, transformers' wants the matrix to weight embeddings.
							lowerCAmelCase =		word_embeddings

							return sd_hf



def    UpperCAmelCase							(  _snake_case    ,			_snake_case				):
							lowerCAmelCase =		64000
							lowerCAmelCase =		config_megatron['''encoder_seq_length''']
							lowerCAmelCase =		config_megatron['''hidden_size''']
							lowerCAmelCase =		config_megatron['''num_layers''']
							lowerCAmelCase =		config_megatron['''num_attention_heads''']
							lowerCAmelCase =		config_megatron['''ffn_hidden_size''']
							lowerCAmelCase =		'''gelu'''
							lowerCAmelCase =		0.1
							lowerCAmelCase =		0.1
							lowerCAmelCase =		0.1
							lowerCAmelCase =		config_megatron['''layernorm_epsilon''']  # 1e-5
							lowerCAmelCase =		config_megatron['''init_method_std''']  # 0.02
							lowerCAmelCase =		config_megatron['''apply_query_key_layer_scaling''']  # True
							lowerCAmelCase =		True
							lowerCAmelCase =		True

							# This identifies the 6.7B (7B) model which uses a different tokenizer
							if config_megatron["hidden_size"] == 4096:
														lowerCAmelCase =		1  # <|endoftext|>
														lowerCAmelCase =		1  # <|endoftext|>
														lowerCAmelCase =		0  # <unk>
							else:
														lowerCAmelCase =		2  # <s>
														lowerCAmelCase =		3  # <|endoftext|>
														lowerCAmelCase =		0  # <pad>

							return config_hf



def    UpperCAmelCase							(  _snake_case				):
							print(_snake_case				)

							lowerCAmelCase =		args.checkpoint_path
							lowerCAmelCase =		args.save_path
							if isfile(_snake_case				):
														raise FileNotFoundError(F"""ERROR! could not find file {checkpoint_path}"""				)

							# Load the model.
							lowerCAmelCase =		torch.load(_snake_case    ,			map_location='''cpu'''				)

							# Load the config.
							lowerCAmelCase =		checkpoint['''hyper_parameters''']['''cfg''']
							lowerCAmelCase =		GPTaConfig()
							lowerCAmelCase =		copy_config(config_hf=_snake_case    ,			config_megatron=_snake_case				)
							lowerCAmelCase =		['''GPT2LMHeadModel''']

							lowerCAmelCase =		checkpoint['''state_dict''']

							# Convert.
							print('''Converting'''				)
							lowerCAmelCase =		convert_megatron_checkpoint(_snake_case    ,			_snake_case				)

							# Print the structure of converted state dict.
							if args.print_checkpoint_structure:
														recursive_print(_snake_case    ,			_snake_case				)

							lowerCAmelCase =		'''GPTSw3Tokenizer'''

							# Store the config to file.
							print('''Saving config'''				)
							config_hf.save_pretrained(_snake_case				)

							# Store the state_dict to file.
							lowerCAmelCase =		os.path.join(_snake_case    ,			'''pytorch_model.bin'''				)
							print(F"""Saving checkpoint to \"{output_checkpoint_file}\""""				)
							torch.save(_snake_case    ,			_snake_case				)


if __name__ == "__main__":
		UpperCAmelCase_    =argparse.ArgumentParser()
		parser.add_argument(
		    """--checkpoint_path""",
		    type=str,
		    required=True,
		    help="""e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000""",
		)
		parser.add_argument("""--save_path""", type=str, required=True, help="""e.g. /home/user/gpt-sw3/hf""")
		parser.add_argument("""--print-checkpoint-structure""", action="""store_true""")
		UpperCAmelCase_    =parser.parse_args()
		main(_args)









from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


UpperCAmelCase_    ={
    """configuration_splinter""": ["""SPLINTER_PRETRAINED_CONFIG_ARCHIVE_MAP""", """SplinterConfig"""],
    """tokenization_splinter""": ["""SplinterTokenizer"""],
}

try:
		if not is_tokenizers_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCAmelCase_    =["""SplinterTokenizerFast"""]

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCAmelCase_    =[
		    """SPLINTER_PRETRAINED_MODEL_ARCHIVE_LIST""",
		    """SplinterForQuestionAnswering""",
		    """SplinterForPreTraining""",
		    """SplinterLayer""",
		    """SplinterModel""",
		    """SplinterPreTrainedModel""",
		]


if TYPE_CHECKING:
		from .configuration_splinter import SPLINTER_PRETRAINED_CONFIG_ARCHIVE_MAP, SplinterConfig
		from .tokenization_splinter import SplinterTokenizer

		try:
				if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_splinter_fast import SplinterTokenizerFast

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_splinter import (
				    SPLINTER_PRETRAINED_MODEL_ARCHIVE_LIST,
				    SplinterForPreTraining,
				    SplinterForQuestionAnswering,
				    SplinterLayer,
				    SplinterModel,
				    SplinterPreTrainedModel,
				)


else:
		import sys

		UpperCAmelCase_    =_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



