



"""simple docstring"""






import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig


def 							A__ (     UpperCamelCase__       ,							UpperCamelCase__       ,							UpperCamelCase__=0    ):


  '''simple docstring'''


  if name is None:
    _SCREAMING_SNAKE_CASE        =			None
  else:
    _SCREAMING_SNAKE_CASE        =			'.' * max(0       ,							spaces - 2    ) + '# {:' + str(50 - spaces    ) + 's}'
    _SCREAMING_SNAKE_CASE        =			fmt.format(lowercase__    )

  # Print and recurse (if needed).
  if isinstance(lowercase__       ,							lowercase__    ):
    if msg is not None:
      print(lowercase__    )
    for k in val.keys():
      recursive_print(lowercase__       ,							val[k]       ,							spaces + 2    )
  elif isinstance(lowercase__       ,							torch.Tensor    ):
    print(lowercase__       ,							''':'''       ,							val.size()    )
  else:
    print(lowercase__       ,							''':'''       ,							lowercase__    )


def 							A__ (     UpperCamelCase__       ,							UpperCamelCase__       ,							UpperCamelCase__       ,							UpperCamelCase__    ):


  '''simple docstring'''


  _SCREAMING_SNAKE_CASE        =			param.size()
  # other versions store [num_heads * num_splits * hidden_size, :]
  _SCREAMING_SNAKE_CASE        =			(num_heads, num_splits, hidden_size) + input_shape[1:]
  _SCREAMING_SNAKE_CASE        =			param.view(*lowercase__    )
  _SCREAMING_SNAKE_CASE        =			param.transpose(0       ,							1    ).contiguous()
  _SCREAMING_SNAKE_CASE        =			param.view(*lowercase__    )
  return param


def 							A__ (     UpperCamelCase__       ,							UpperCamelCase__    ):


  '''simple docstring'''


  _SCREAMING_SNAKE_CASE        =			config.n_positions
  _SCREAMING_SNAKE_CASE        =			config.n_layer
  _SCREAMING_SNAKE_CASE        =			config.vocab_size
  _SCREAMING_SNAKE_CASE        =			config.n_head
  _SCREAMING_SNAKE_CASE        =			config.n_embd // config.n_head

  _SCREAMING_SNAKE_CASE        =			sd_megatron['model.language_model.embedding.word_embeddings.weight'][:vocab_size, :]
  _SCREAMING_SNAKE_CASE        =			{
      'transformer.wte.weight': word_embeddings,
      'transformer.wpe.weight': sd_megatron['model.language_model.embedding.position_embeddings.weight'],
      'transformer.ln_f.weight': sd_megatron['model.language_model.encoder.final_layernorm.weight'],
      'transformer.ln_f.bias': sd_megatron['model.language_model.encoder.final_layernorm.bias'],
  }

  _SCREAMING_SNAKE_CASE        =			'model.language_model.encoder.layers.'
  for i in range(lowercase__    ):
    _SCREAMING_SNAKE_CASE        =			torch.tril(torch.ones((n_positions, n_positions)       ,							dtype=torch.bool    )    )
    _SCREAMING_SNAKE_CASE        =			causal_mask.view(1       ,							1       ,							lowercase__       ,							lowercase__    )
    _SCREAMING_SNAKE_CASE        =			causal_mask
    _SCREAMING_SNAKE_CASE        =			torch.tensor(-1e4       ,							dtype=torch.bfloataa    )

    _SCREAMING_SNAKE_CASE        =			sd_megatron[F'''{pf}{i}.input_layernorm.weight''']
    _SCREAMING_SNAKE_CASE        =			sd_megatron[F'''{pf}{i}.input_layernorm.bias''']

    _SCREAMING_SNAKE_CASE        =			sd_megatron[F'''{pf}{i}.self_attention.query_key_value.weight''']
    _SCREAMING_SNAKE_CASE        =			fix_query_key_value_ordering(lowercase__       ,							3       ,							lowercase__       ,							lowercase__    )
    _SCREAMING_SNAKE_CASE        =			vala.transpose(0       ,							1    ).contiguous()

    _SCREAMING_SNAKE_CASE        =			sd_megatron[F'''{pf}{i}.self_attention.query_key_value.bias''']
    _SCREAMING_SNAKE_CASE        =			fix_query_key_value_ordering(lowercase__       ,							3       ,							lowercase__       ,							lowercase__    )
    _SCREAMING_SNAKE_CASE        =			vala

    _SCREAMING_SNAKE_CASE        =			sd_megatron[F'''{pf}{i}.self_attention.dense.weight'''].transpose(
        0       ,							1    )
    _SCREAMING_SNAKE_CASE        =			sd_megatron[F'''{pf}{i}.self_attention.dense.bias''']
    _SCREAMING_SNAKE_CASE        =			sd_megatron[F'''{pf}{i}.post_attention_layernorm.weight''']
    _SCREAMING_SNAKE_CASE        =			sd_megatron[F'''{pf}{i}.post_attention_layernorm.bias''']
    _SCREAMING_SNAKE_CASE        =			sd_megatron[F'''{pf}{i}.mlp.dense_h_to_4h.weight'''].transpose(0       ,							1    )
    _SCREAMING_SNAKE_CASE        =			sd_megatron[F'''{pf}{i}.mlp.dense_h_to_4h.bias''']
    _SCREAMING_SNAKE_CASE        =			sd_megatron[F'''{pf}{i}.mlp.dense_4h_to_h.weight'''].transpose(
        0       ,							1    )
    _SCREAMING_SNAKE_CASE        =			sd_megatron[F'''{pf}{i}.mlp.dense_4h_to_h.bias''']

  # For LM head, transformers' wants the matrix to weight embeddings.
  _SCREAMING_SNAKE_CASE        =			word_embeddings

  return sd_hf


def 							A__ (     UpperCamelCase__       ,							UpperCamelCase__    ):


  '''simple docstring'''


  _SCREAMING_SNAKE_CASE        =			64_000
  _SCREAMING_SNAKE_CASE        =			config_megatron['encoder_seq_length']
  _SCREAMING_SNAKE_CASE        =			config_megatron['hidden_size']
  _SCREAMING_SNAKE_CASE        =			config_megatron['num_layers']
  _SCREAMING_SNAKE_CASE        =			config_megatron['num_attention_heads']
  _SCREAMING_SNAKE_CASE        =			config_megatron['ffn_hidden_size']
  _SCREAMING_SNAKE_CASE        =			'gelu'
  _SCREAMING_SNAKE_CASE        =			0.1
  _SCREAMING_SNAKE_CASE        =			0.1
  _SCREAMING_SNAKE_CASE        =			0.1
  _SCREAMING_SNAKE_CASE        =			config_megatron['layernorm_epsilon']  # 1e-5
  _SCREAMING_SNAKE_CASE        =			config_megatron['init_method_std']  # 0.02
  _SCREAMING_SNAKE_CASE        =			config_megatron['apply_query_key_layer_scaling']  # True
  _SCREAMING_SNAKE_CASE        =			True
  _SCREAMING_SNAKE_CASE        =			True

  # This identifies the 6.7B (7B) model which uses a different tokenizer
  if config_megatron["hidden_size"] == 4_096:
    _SCREAMING_SNAKE_CASE        =			1  # <|endoftext|>
    _SCREAMING_SNAKE_CASE        =			1  # <|endoftext|>
    _SCREAMING_SNAKE_CASE        =			0  # <unk>
  else:
    _SCREAMING_SNAKE_CASE        =			2  # <s>
    _SCREAMING_SNAKE_CASE        =			3  # <|endoftext|>
    _SCREAMING_SNAKE_CASE        =			0  # <pad>

  return config_hf


def 							A__ (     UpperCamelCase__    ):


  '''simple docstring'''


  print(lowercase__    )

  _SCREAMING_SNAKE_CASE        =			args.checkpoint_path
  _SCREAMING_SNAKE_CASE        =			args.save_path
  if isfile(lowercase__    ):
    raise FileNotFoundError(F'''ERROR! could not find file {checkpoint_path}'''    )

  # Load the model.
  _SCREAMING_SNAKE_CASE        =			torch.load(lowercase__       ,							map_location='''cpu'''    )

  # Load the config.
  _SCREAMING_SNAKE_CASE        =			checkpoint['hyper_parameters']['cfg']
  _SCREAMING_SNAKE_CASE        =			GPTaConfig()
  _SCREAMING_SNAKE_CASE        =			copy_config(config_hf=lowercase__       ,							config_megatron=lowercase__    )
  _SCREAMING_SNAKE_CASE        =			['GPT2LMHeadModel']

  _SCREAMING_SNAKE_CASE        =			checkpoint['state_dict']

  # Convert.
  print('''Converting'''    )
  _SCREAMING_SNAKE_CASE        =			convert_megatron_checkpoint(lowercase__       ,							lowercase__    )

  # Print the structure of converted state dict.
  if args.print_checkpoint_structure:
    recursive_print(lowercase__       ,							lowercase__    )

  _SCREAMING_SNAKE_CASE        =			'GPTSw3Tokenizer'

  # Store the config to file.
  print('''Saving config'''    )
  config_hf.save_pretrained(lowercase__    )

  # Store the state_dict to file.
  _SCREAMING_SNAKE_CASE        =			os.path.join(lowercase__       ,							'''pytorch_model.bin'''    )
  print(F'''Saving checkpoint to \"{output_checkpoint_file}\"'''    )
  torch.save(lowercase__       ,							lowercase__    )


if __name__ == "__main__":
    lowerCamelCase				:     Union[str, Any]								=  argparse.ArgumentParser()
    parser.add_argument(
        """--checkpoint_path""",
        type=str,
        required=True,
        help="""e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000""",
    )
    parser.add_argument("""--save_path""", type=str, required=True, help="""e.g. /home/user/gpt-sw3/hf""")
    parser.add_argument("""--print-checkpoint-structure""", action="""store_true""")
    lowerCamelCase				:     Any								=  parser.parse_args()
    main(_args)




import math
def 			__SCREAMING_SNAKE_CASE							(					lowercase__	:int					)							->	list:
						if n <= 0:
												raise ValueError('Only positive integers have prime factors'					)
						__magic_name__					:				Union[str, Any]				=   []
						while n % 2 == 0:
												pf.append(2					)
												__magic_name__					:				Union[str, Any]				=   int(n / 2					)
						for i in range(3   ,    int(math.sqrt(lowercase__					)					) + 1   ,    2					):
												while n % i == 0:
																		pf.append(lowercase__					)
																		__magic_name__					:				List[str]				=   int(n / i					)
						if n > 2:
												pf.append(lowercase__					)
						return pf
def 			__SCREAMING_SNAKE_CASE							(					lowercase__	:int					)							->	int:
						if n <= 0:
												raise ValueError('Only positive numbers are accepted'					)
						__magic_name__					:				Union[str, Any]				=   1
						__magic_name__					:				Tuple				=   1
						while n % 2 == 0:
												temp += 1
												__magic_name__					:				Any				=   int(n / 2					)
						div *= temp
						for i in range(3   ,    int(math.sqrt(lowercase__					)					) + 1   ,    2					):
												__magic_name__					:				List[Any]				=   1
												while n % i == 0:
																		temp += 1
																		__magic_name__					:				Any				=   int(n / i					)
												div *= temp
						if n > 1:
												div *= 2
						return div
def 			__SCREAMING_SNAKE_CASE							(					lowercase__	:int					)							->	int:
						if n <= 0:
												raise ValueError('Only positive numbers are accepted'					)
						__magic_name__					:				int				=   1
						__magic_name__					:				List[str]				=   1
						while n % 2 == 0:
												temp += 1
												__magic_name__					:				List[Any]				=   int(n / 2					)
						if temp > 1:
												s *= (2**temp - 1) / (2 - 1)
						for i in range(3   ,    int(math.sqrt(lowercase__					)					) + 1   ,    2					):
												__magic_name__					:				int				=   1
												while n % i == 0:
																		temp += 1
																		__magic_name__					:				Optional[Any]				=   int(n / i					)
												if temp > 1:
																		s *= (i**temp - 1) / (i - 1)
						return int(lowercase__					)
def 			__SCREAMING_SNAKE_CASE							(					lowercase__	:int					)							->	int:
						__magic_name__					:				Dict				=   n
						for x in set(prime_factors(lowercase__					)					):
												s *= (x - 1) / x
						return int(lowercase__					)


if __name__ == "__main__":
			import doctest

			doctest.testmod()


