


import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig

def 		_UpperCAmelCase							(     SCREAMING_SNAKE_CASE__     :   Tuple     ,	SCREAMING_SNAKE_CASE__     :   List[Any]     ,	SCREAMING_SNAKE_CASE__     :   Union[str, Any]=0							):
     # Format the message.
     if name is None:
          __UpperCamelCase       				=None
     else:
          __UpperCamelCase       				='.' * max(0     ,	spaces - 2							) + '# {:' + str(50 - spaces							) + 's}'
          __UpperCamelCase       				=fmt.format(SCREAMING_SNAKE_CASE__							)

     # Print and recurse (if needed).
     if isinstance(SCREAMING_SNAKE_CASE__     ,	SCREAMING_SNAKE_CASE__							):
          if msg is not None:
               print(SCREAMING_SNAKE_CASE__							)
          for k in val.keys():
               recursive_print(SCREAMING_SNAKE_CASE__     ,	val[k]     ,	spaces + 2							)
     elif isinstance(SCREAMING_SNAKE_CASE__     ,	torch.Tensor							):
          print(SCREAMING_SNAKE_CASE__     ,	':'     ,	val.size()							)
     else:
          print(SCREAMING_SNAKE_CASE__     ,	':'     ,	SCREAMING_SNAKE_CASE__							)

def 		_UpperCAmelCase							(     SCREAMING_SNAKE_CASE__     :   Dict     ,	SCREAMING_SNAKE_CASE__     :   int     ,	SCREAMING_SNAKE_CASE__     :   Optional[int]     ,	SCREAMING_SNAKE_CASE__     :   List[str]							):
     # Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
     # for compatibility with later versions of NVIDIA Megatron-LM.
     # The inverse operation is performed inside Megatron-LM to read checkpoints:
     # https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
     # If param is the weight tensor of the self-attention block, the returned tensor
     # will have to be transposed one more time to be read by HuggingFace GPT2.
     __UpperCamelCase       				=param.size()
     # other versions store [num_heads * num_splits * hidden_size, :]
     __UpperCamelCase       				=(num_heads, num_splits, hidden_size) + input_shape[1:]
     __UpperCamelCase       				=param.view(*SCREAMING_SNAKE_CASE__							)
     __UpperCamelCase       				=param.transpose(0     ,	1							).contiguous()
     __UpperCamelCase       				=param.view(*SCREAMING_SNAKE_CASE__							)
     return param

def 		_UpperCAmelCase							(     SCREAMING_SNAKE_CASE__     :   int     ,	SCREAMING_SNAKE_CASE__     :   Optional[Any]							):
     __UpperCamelCase       				=config.n_positions
     __UpperCamelCase       				=config.n_layer
     __UpperCamelCase       				=config.vocab_size
     __UpperCamelCase       				=config.n_head
     __UpperCamelCase       				=config.n_embd // config.n_head

     __UpperCamelCase       				=sd_megatron['model.language_model.embedding.word_embeddings.weight'][:vocab_size, :]
     __UpperCamelCase       				={
         'transformer.wte.weight': word_embeddings,
         'transformer.wpe.weight': sd_megatron['model.language_model.embedding.position_embeddings.weight'],
         'transformer.ln_f.weight': sd_megatron['model.language_model.encoder.final_layernorm.weight'],
         'transformer.ln_f.bias': sd_megatron['model.language_model.encoder.final_layernorm.bias'],
     }

     __UpperCamelCase       				='model.language_model.encoder.layers.'
     for i in range(SCREAMING_SNAKE_CASE__							):
          __UpperCamelCase       				=torch.tril(torch.ones((n_positions, n_positions)     ,	dtype=torch.bool							)							)
          __UpperCamelCase       				=causal_mask.view(1     ,	1     ,	SCREAMING_SNAKE_CASE__     ,	SCREAMING_SNAKE_CASE__							)
          __UpperCamelCase       				=causal_mask
          __UpperCamelCase       				=torch.tensor(-1E4     ,	dtype=torch.bfloataa							)

          __UpperCamelCase       				=sd_megatron[F'{pf}{i}.input_layernorm.weight']
          __UpperCamelCase       				=sd_megatron[F'{pf}{i}.input_layernorm.bias']

          __UpperCamelCase       				=sd_megatron[F'{pf}{i}.self_attention.query_key_value.weight']
          __UpperCamelCase       				=fix_query_key_value_ordering(SCREAMING_SNAKE_CASE__     ,	3     ,	SCREAMING_SNAKE_CASE__     ,	SCREAMING_SNAKE_CASE__							)
          __UpperCamelCase       				=vala.transpose(0     ,	1							).contiguous()

          __UpperCamelCase       				=sd_megatron[F'{pf}{i}.self_attention.query_key_value.bias']
          __UpperCamelCase       				=fix_query_key_value_ordering(SCREAMING_SNAKE_CASE__     ,	3     ,	SCREAMING_SNAKE_CASE__     ,	SCREAMING_SNAKE_CASE__							)
          __UpperCamelCase       				=vala

          __UpperCamelCase       				=sd_megatron[F'{pf}{i}.self_attention.dense.weight'].transpose(
              0     ,	1							)
          __UpperCamelCase       				=sd_megatron[F'{pf}{i}.self_attention.dense.bias']
          __UpperCamelCase       				=sd_megatron[F'{pf}{i}.post_attention_layernorm.weight']
          __UpperCamelCase       				=sd_megatron[F'{pf}{i}.post_attention_layernorm.bias']
          __UpperCamelCase       				=sd_megatron[F'{pf}{i}.mlp.dense_h_to_4h.weight'].transpose(0     ,	1							)
          __UpperCamelCase       				=sd_megatron[F'{pf}{i}.mlp.dense_h_to_4h.bias']
          __UpperCamelCase       				=sd_megatron[F'{pf}{i}.mlp.dense_4h_to_h.weight'].transpose(
              0     ,	1							)
          __UpperCamelCase       				=sd_megatron[F'{pf}{i}.mlp.dense_4h_to_h.bias']

     # For LM head, transformers' wants the matrix to weight embeddings.
     __UpperCamelCase       				=word_embeddings

     return sd_hf

def 		_UpperCAmelCase							(     SCREAMING_SNAKE_CASE__     :   Tuple     ,	SCREAMING_SNAKE_CASE__     :   Optional[Any]							):
     __UpperCamelCase       				=6_40_00
     __UpperCamelCase       				=config_megatron['encoder_seq_length']
     __UpperCamelCase       				=config_megatron['hidden_size']
     __UpperCamelCase       				=config_megatron['num_layers']
     __UpperCamelCase       				=config_megatron['num_attention_heads']
     __UpperCamelCase       				=config_megatron['ffn_hidden_size']
     __UpperCamelCase       				='gelu'
     __UpperCamelCase       				=0.1
     __UpperCamelCase       				=0.1
     __UpperCamelCase       				=0.1
     __UpperCamelCase       				=config_megatron['layernorm_epsilon']  # 1e-5
     __UpperCamelCase       				=config_megatron['init_method_std']  # 0.02
     __UpperCamelCase       				=config_megatron['apply_query_key_layer_scaling']  # True
     __UpperCamelCase       				=True
     __UpperCamelCase       				=True

     # This identifies the 6.7B (7B) model which uses a different tokenizer
     if config_megatron["hidden_size"] == 40_96:
          __UpperCamelCase       				=1  # <|endoftext|>
          __UpperCamelCase       				=1  # <|endoftext|>
          __UpperCamelCase       				=0  # <unk>
     else:
          __UpperCamelCase       				=2  # <s>
          __UpperCamelCase       				=3  # <|endoftext|>
          __UpperCamelCase       				=0  # <pad>

     return config_hf

def 		_UpperCAmelCase							(     SCREAMING_SNAKE_CASE__     :   Dict							):
     print(SCREAMING_SNAKE_CASE__							)

     __UpperCamelCase       				=args.checkpoint_path
     __UpperCamelCase       				=args.save_path
     if isfile(SCREAMING_SNAKE_CASE__							):
          raise FileNotFoundError(F'ERROR! could not find file {checkpoint_path}'							)

     # Load the model.
     __UpperCamelCase       				=torch.load(SCREAMING_SNAKE_CASE__     ,	map_location='cpu'							)

     # Load the config.
     __UpperCamelCase       				=checkpoint['hyper_parameters']['cfg']
     __UpperCamelCase       				=GPTaConfig()
     __UpperCamelCase       				=copy_config(config_hf=SCREAMING_SNAKE_CASE__     ,	config_megatron=SCREAMING_SNAKE_CASE__							)
     __UpperCamelCase       				=['GPT2LMHeadModel']

     __UpperCamelCase       				=checkpoint['state_dict']

     # Convert.
     print('Converting'							)
     __UpperCamelCase       				=convert_megatron_checkpoint(SCREAMING_SNAKE_CASE__     ,	SCREAMING_SNAKE_CASE__							)

     # Print the structure of converted state dict.
     if args.print_checkpoint_structure:
          recursive_print(SCREAMING_SNAKE_CASE__     ,	SCREAMING_SNAKE_CASE__							)

     __UpperCamelCase       				='GPTSw3Tokenizer'

     # Store the config to file.
     print('Saving config'							)
     config_hf.save_pretrained(SCREAMING_SNAKE_CASE__							)

     # Store the state_dict to file.
     __UpperCamelCase       				=os.path.join(SCREAMING_SNAKE_CASE__     ,	'pytorch_model.bin'							)
     print(F'Saving checkpoint to "{output_checkpoint_file}"'							)
     torch.save(SCREAMING_SNAKE_CASE__     ,	SCREAMING_SNAKE_CASE__							)


if __name__ == "__main__":
   _A									=		argparse.ArgumentParser()
   parser.add_argument(
       '--checkpoint_path',
       type=str,
       required=True,
       help='e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000',
   )
   parser.add_argument('--save_path', type=str, required=True, help='e.g. /home/user/gpt-sw3/hf')
   parser.add_argument('--print-checkpoint-structure', action='store_true')
   _A									=		parser.parse_args()
   main(_args)









def 		_UpperCAmelCase							(     SCREAMING_SNAKE_CASE__     :   list     ,	SCREAMING_SNAKE_CASE__     :   int     ,	SCREAMING_SNAKE_CASE__     :   int     ,	SCREAMING_SNAKE_CASE__     :   int							):
     __UpperCamelCase       				=int((lower_bound + upper_bound) // 2							)
     if array[r] == value:
          return r
     if lower_bound >= upper_bound:
          return -1
     if array[r] < value:
          return binary_search(SCREAMING_SNAKE_CASE__     ,	r + 1     ,	SCREAMING_SNAKE_CASE__     ,	SCREAMING_SNAKE_CASE__							)
     else:
          return binary_search(SCREAMING_SNAKE_CASE__     ,	SCREAMING_SNAKE_CASE__     ,	r - 1     ,	SCREAMING_SNAKE_CASE__							)

def 		_UpperCAmelCase							(     SCREAMING_SNAKE_CASE__     :   int     ,	SCREAMING_SNAKE_CASE__     :   list							):
     __UpperCamelCase       				=0
     if matrix[index][0] == value:
          return [index, 0]
     while index < len(SCREAMING_SNAKE_CASE__							) and matrix[index][0] < value:
          __UpperCamelCase       				=binary_search(matrix[index]     ,	0     ,	len(matrix[index]							) - 1     ,	SCREAMING_SNAKE_CASE__							)
          if r != -1:
               return [index, r]
          index += 1
     return [-1, -1]


if __name__ == "__main__":
   import doctest

   doctest.testmod()





