

import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig



def  snake_case_(_UpperCamelCase					,			_UpperCamelCase					,			_UpperCamelCase=0	)				->      Dict:



  """simple docstring"""






  if name is None:
    _snake_case		=				None
  else:
    _snake_case		=				'''.''' * max(0					,			spaces - 2	) + '''# {:''' + str(50 - spaces	) + '''s}'''
    _snake_case		=				fmt.format(_UpperCamelCase	)

  # Print and recurse (if needed).
  if isinstance(_UpperCamelCase					,			_UpperCamelCase	):
    if msg is not None:
      print(_UpperCamelCase	)
    for k in val.keys():
      recursive_print(_UpperCamelCase					,			val[k]					,			spaces + 2	)
  elif isinstance(_UpperCamelCase					,			torch.Tensor	):
    print(_UpperCamelCase					,			''':'''					,			val.size()	)
  else:
    print(_UpperCamelCase					,			''':'''					,			_UpperCamelCase	)



def  snake_case_(_UpperCamelCase					,			_UpperCamelCase					,			_UpperCamelCase					,			_UpperCamelCase	)				->      Dict:



  """simple docstring"""






  _snake_case		=				param.size()
  # other versions store [num_heads * num_splits * hidden_size, :]
  _snake_case		=				(num_heads, num_splits, hidden_size) + input_shape[1:]
  _snake_case		=				param.view(*_UpperCamelCase	)
  _snake_case		=				param.transpose(0					,			1	).contiguous()
  _snake_case		=				param.view(*_UpperCamelCase	)
  return param



def  snake_case_(_UpperCamelCase					,			_UpperCamelCase	)				->      Optional[Any]:



  """simple docstring"""






  _snake_case		=				config.n_positions
  _snake_case		=				config.n_layer
  _snake_case		=				config.vocab_size
  _snake_case		=				config.n_head
  _snake_case		=				config.n_embd // config.n_head

  _snake_case		=				sd_megatron['''model.language_model.embedding.word_embeddings.weight'''][:vocab_size, :]
  _snake_case		=				{
      '''transformer.wte.weight''': word_embeddings,
      '''transformer.wpe.weight''': sd_megatron['''model.language_model.embedding.position_embeddings.weight'''],
      '''transformer.ln_f.weight''': sd_megatron['''model.language_model.encoder.final_layernorm.weight'''],
      '''transformer.ln_f.bias''': sd_megatron['''model.language_model.encoder.final_layernorm.bias'''],
  }

  _snake_case		=				'''model.language_model.encoder.layers.'''
  for i in range(_UpperCamelCase	):
    _snake_case		=				torch.tril(torch.ones((n_positions, n_positions)					,			dtype=torch.bool	)	)
    _snake_case		=				causal_mask.view(1					,			1					,			_UpperCamelCase					,			_UpperCamelCase	)
    _snake_case		=				causal_mask
    _snake_case		=				torch.tensor(-1E4					,			dtype=torch.bfloataa	)

    _snake_case		=				sd_megatron[F"""{pf}{i}.input_layernorm.weight"""]
    _snake_case		=				sd_megatron[F"""{pf}{i}.input_layernorm.bias"""]

    _snake_case		=				sd_megatron[F"""{pf}{i}.self_attention.query_key_value.weight"""]
    _snake_case		=				fix_query_key_value_ordering(_UpperCamelCase					,			3					,			_UpperCamelCase					,			_UpperCamelCase	)
    _snake_case		=				vala.transpose(0					,			1	).contiguous()

    _snake_case		=				sd_megatron[F"""{pf}{i}.self_attention.query_key_value.bias"""]
    _snake_case		=				fix_query_key_value_ordering(_UpperCamelCase					,			3					,			_UpperCamelCase					,			_UpperCamelCase	)
    _snake_case		=				vala

    _snake_case		=				sd_megatron[F"""{pf}{i}.self_attention.dense.weight"""].transpose(
        0					,			1	)
    _snake_case		=				sd_megatron[F"""{pf}{i}.self_attention.dense.bias"""]
    _snake_case		=				sd_megatron[F"""{pf}{i}.post_attention_layernorm.weight"""]
    _snake_case		=				sd_megatron[F"""{pf}{i}.post_attention_layernorm.bias"""]
    _snake_case		=				sd_megatron[F"""{pf}{i}.mlp.dense_h_to_4h.weight"""].transpose(0					,			1	)
    _snake_case		=				sd_megatron[F"""{pf}{i}.mlp.dense_h_to_4h.bias"""]
    _snake_case		=				sd_megatron[F"""{pf}{i}.mlp.dense_4h_to_h.weight"""].transpose(
        0					,			1	)
    _snake_case		=				sd_megatron[F"""{pf}{i}.mlp.dense_4h_to_h.bias"""]

  # For LM head, transformers' wants the matrix to weight embeddings.
  _snake_case		=				word_embeddings

  return sd_hf



def  snake_case_(_UpperCamelCase					,			_UpperCamelCase	)				->      Dict:



  """simple docstring"""






  _snake_case		=				64_000
  _snake_case		=				config_megatron['''encoder_seq_length''']
  _snake_case		=				config_megatron['''hidden_size''']
  _snake_case		=				config_megatron['''num_layers''']
  _snake_case		=				config_megatron['''num_attention_heads''']
  _snake_case		=				config_megatron['''ffn_hidden_size''']
  _snake_case		=				'''gelu'''
  _snake_case		=				0.1
  _snake_case		=				0.1
  _snake_case		=				0.1
  _snake_case		=				config_megatron['''layernorm_epsilon''']  # 1e-5
  _snake_case		=				config_megatron['''init_method_std''']  # 0.02
  _snake_case		=				config_megatron['''apply_query_key_layer_scaling''']  # True
  _snake_case		=				True
  _snake_case		=				True

  # This identifies the 6.7B (7B) model which uses a different tokenizer
  if config_megatron["hidden_size"] == 4_096:
    _snake_case		=				1  # <|endoftext|>
    _snake_case		=				1  # <|endoftext|>
    _snake_case		=				0  # <unk>
  else:
    _snake_case		=				2  # <s>
    _snake_case		=				3  # <|endoftext|>
    _snake_case		=				0  # <pad>

  return config_hf



def  snake_case_(_UpperCamelCase	)				->      int:



  """simple docstring"""






  print(_UpperCamelCase	)

  _snake_case		=				args.checkpoint_path
  _snake_case		=				args.save_path
  if isfile(_UpperCamelCase	):
    raise FileNotFoundError(F"""ERROR! could not find file {checkpoint_path}"""	)

  # Load the model.
  _snake_case		=				torch.load(_UpperCamelCase					,			map_location='''cpu'''	)

  # Load the config.
  _snake_case		=				checkpoint['''hyper_parameters''']['''cfg''']
  _snake_case		=				GPTaConfig()
  _snake_case		=				copy_config(config_hf=_UpperCamelCase					,			config_megatron=_UpperCamelCase	)
  _snake_case		=				['''GPT2LMHeadModel''']

  _snake_case		=				checkpoint['''state_dict''']

  # Convert.
  print('''Converting'''	)
  _snake_case		=				convert_megatron_checkpoint(_UpperCamelCase					,			_UpperCamelCase	)

  # Print the structure of converted state dict.
  if args.print_checkpoint_structure:
    recursive_print(_UpperCamelCase					,			_UpperCamelCase	)

  _snake_case		=				'''GPTSw3Tokenizer'''

  # Store the config to file.
  print('''Saving config'''	)
  config_hf.save_pretrained(_UpperCamelCase	)

  # Store the state_dict to file.
  _snake_case		=				os.path.join(_UpperCamelCase					,			'''pytorch_model.bin'''	)
  print(F"""Saving checkpoint to \"{output_checkpoint_file}\""""	)
  torch.save(_UpperCamelCase					,			_UpperCamelCase	)


if __name__ == "__main__":
 __A     						=      argparse.ArgumentParser()
 parser.add_argument(
     '''--checkpoint_path''',
     type=str,
     required=True,
     help='''e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000''',
 )
 parser.add_argument('''--save_path''', type=str, required=True, help='''e.g. /home/user/gpt-sw3/hf''')
 parser.add_argument('''--print-checkpoint-structure''', action='''store_true''')
 __A     						=      parser.parse_args()
 main(_args)




import functools
from math import sqrt
from time import time



def  snake_case_(_UpperCamelCase					,			*_UpperCamelCase					,			**_UpperCamelCase	)				->      str:



  """simple docstring"""






  _snake_case		=				time()
  _snake_case		=				func(*_UpperCamelCase					,			**_UpperCamelCase	)
  _snake_case		=				time()
  if int(end - start	) > 0:
    print(F"""{func.__name__} runtime: {(end - start):0.4f} s"""	)
  else:
    print(F"""{func.__name__} runtime: {(end - start) * 1_000:0.4f} ms"""	)
  return output



def  snake_case_(_UpperCamelCase	)				->      list[int]:



  """simple docstring"""






  if n < 0:
    raise Exception('''n is negative'''	)
  if n == 0:
    return [0]
  _snake_case		=				[0, 1]
  for _ in range(n - 1	):
    fib.append(fib[-1] + fib[-2]	)
  return fib



def  snake_case_(_UpperCamelCase	)				->      list[int]:



  """simple docstring"""






  def fib_recursive_term(_UpperCamelCase	) -> int:
    if i < 0:
      raise Exception('''n is negative'''	)
    if i < 2:
      return i
    return fib_recursive_term(i - 1	) + fib_recursive_term(i - 2	)

  if n < 0:
    raise Exception('''n is negative'''	)
  return [fib_recursive_term(_UpperCamelCase	) for i in range(n + 1	)]



def  snake_case_(_UpperCamelCase	)				->      list[int]:



  """simple docstring"""






  @functools.cache
  def fib_recursive_term(_UpperCamelCase	) -> int:
    if i < 0:
      raise Exception('''n is negative'''	)
    if i < 2:
      return i
    return fib_recursive_term(i - 1	) + fib_recursive_term(i - 2	)

  if n < 0:
    raise Exception('''n is negative'''	)
  return [fib_recursive_term(_UpperCamelCase	) for i in range(n + 1	)]



def  snake_case_(_UpperCamelCase	)				->      list[int]:



  """simple docstring"""






  if n < 0:
    raise Exception('''n is negative'''	)
  # Cache must be outside recursuive function
  # other it will reset every time it calls itself.
  _snake_case		=				{0: 0, 1: 1, 2: 1}  # Prefilled cache

  def rec_fn_memoized(_UpperCamelCase	) -> int:
    if num in cache:
      return cache[num]

    _snake_case		=				rec_fn_memoized(num - 1	) + rec_fn_memoized(num - 2	)
    _snake_case		=				value
    return value

  return [rec_fn_memoized(_UpperCamelCase	) for i in range(n + 1	)]



def  snake_case_(_UpperCamelCase	)				->      list[int]:



  """simple docstring"""






  if n < 0:
    raise Exception('''n is negative'''	)
  if n >= 1_475:
    raise Exception('''n is too large'''	)
  _snake_case		=				sqrt(5	)
  _snake_case		=				(1 + sqrt_a) / 2
  return [round(phi**i / sqrt_a	) for i in range(n + 1	)]


if __name__ == "__main__":
 __A     						=      30
 time_func(fib_iterative, num)
 time_func(fib_recursive, num)  # Around 3s runtime
 time_func(fib_recursive_cached, num)  # Around 0ms runtime
 time_func(fib_memoization, num)
 time_func(fib_binet, num)

