



'''simple docstring'''






def 				_SCREAMING_SNAKE_CASE(					snake_case_    :     int = 10_00  )		->int:






							'''simple docstring'''
							_lowercase							:		Union[str, Any]				=   0
							for i in range(a__  ):
														if i % 3 == 0 or i % 5 == 0:
																					result += i
							return result


if __name__ == "__main__":
						print(f'''{solution() = }''')






"""simple docstring"""

import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig





def 					__UpperCamelCase (     a__:			List[Any]			,						a__:			int			,						a__:			Dict=0      )					->      Any:




					"""simple docstring"""



					if name is None:
										__lowercase        =      None
					else:
										__lowercase        =      '''.''' * max(0			,						spaces - 2      ) + '''# {:''' + str(50 - spaces      ) + '''s}'''
										__lowercase        =      fmt.format(a__      )

					# Print and recurse (if needed).
					if isinstance(a__			,						a__      ):
										if msg is not None:
															print(a__      )
										for k in val.keys():
															recursive_print(a__			,						val[k]			,						spaces + 2      )
					elif isinstance(a__			,						torch.Tensor      ):
										print(a__			,						''':'''			,						val.size()      )
					else:
										print(a__			,						''':'''			,						a__      )





def 					__UpperCamelCase (     a__:			Optional[int]			,						a__:			Tuple			,						a__:			List[Any]			,						a__:			List[str]      )					->      str:




					"""simple docstring"""



					__lowercase        =      param.size()
					# other versions store [num_heads * num_splits * hidden_size, :]
					__lowercase        =      (num_heads, num_splits, hidden_size) + input_shape[1:]
					__lowercase        =      param.view(*a__      )
					__lowercase        =      param.transpose(0			,						1      ).contiguous()
					__lowercase        =      param.view(*a__      )
					return param





def 					__UpperCamelCase (     a__:			Tuple			,						a__:			Optional[int]      )					->      List[str]:




					"""simple docstring"""



					__lowercase        =      config.n_positions
					__lowercase        =      config.n_layer
					__lowercase        =      config.vocab_size
					__lowercase        =      config.n_head
					__lowercase        =      config.n_embd // config.n_head

					__lowercase        =      sd_megatron['''model.language_model.embedding.word_embeddings.weight'''][:vocab_size, :]
					__lowercase        =      {
					    '''transformer.wte.weight''': word_embeddings,
					    '''transformer.wpe.weight''': sd_megatron['''model.language_model.embedding.position_embeddings.weight'''],
					    '''transformer.ln_f.weight''': sd_megatron['''model.language_model.encoder.final_layernorm.weight'''],
					    '''transformer.ln_f.bias''': sd_megatron['''model.language_model.encoder.final_layernorm.bias'''],
					}

					__lowercase        =      '''model.language_model.encoder.layers.'''
					for i in range(a__      ):
										__lowercase        =      torch.tril(torch.ones((n_positions, n_positions)			,						dtype=torch.bool      )      )
										__lowercase        =      causal_mask.view(1			,						1			,						a__			,						a__      )
										__lowercase        =      causal_mask
										__lowercase        =      torch.tensor(-1e4			,						dtype=torch.bfloataa      )

										__lowercase        =      sd_megatron[f'''{pf}{i}.input_layernorm.weight''']
										__lowercase        =      sd_megatron[f'''{pf}{i}.input_layernorm.bias''']

										__lowercase        =      sd_megatron[f'''{pf}{i}.self_attention.query_key_value.weight''']
										__lowercase        =      fix_query_key_value_ordering(a__			,						3			,						a__			,						a__      )
										__lowercase        =      vala.transpose(0			,						1      ).contiguous()

										__lowercase        =      sd_megatron[f'''{pf}{i}.self_attention.query_key_value.bias''']
										__lowercase        =      fix_query_key_value_ordering(a__			,						3			,						a__			,						a__      )
										__lowercase        =      vala

										__lowercase        =      sd_megatron[f'''{pf}{i}.self_attention.dense.weight'''].transpose(
										    0			,						1      )
										__lowercase        =      sd_megatron[f'''{pf}{i}.self_attention.dense.bias''']
										__lowercase        =      sd_megatron[f'''{pf}{i}.post_attention_layernorm.weight''']
										__lowercase        =      sd_megatron[f'''{pf}{i}.post_attention_layernorm.bias''']
										__lowercase        =      sd_megatron[f'''{pf}{i}.mlp.dense_h_to_4h.weight'''].transpose(0			,						1      )
										__lowercase        =      sd_megatron[f'''{pf}{i}.mlp.dense_h_to_4h.bias''']
										__lowercase        =      sd_megatron[f'''{pf}{i}.mlp.dense_4h_to_h.weight'''].transpose(
										    0			,						1      )
										__lowercase        =      sd_megatron[f'''{pf}{i}.mlp.dense_4h_to_h.bias''']

					# For LM head, transformers' wants the matrix to weight embeddings.
					__lowercase        =      word_embeddings

					return sd_hf





def 					__UpperCamelCase (     a__:			Any			,						a__:			List[Any]      )					->      List[Any]:




					"""simple docstring"""



					__lowercase        =      6_40_00
					__lowercase        =      config_megatron['''encoder_seq_length''']
					__lowercase        =      config_megatron['''hidden_size''']
					__lowercase        =      config_megatron['''num_layers''']
					__lowercase        =      config_megatron['''num_attention_heads''']
					__lowercase        =      config_megatron['''ffn_hidden_size''']
					__lowercase        =      '''gelu'''
					__lowercase        =      0.1
					__lowercase        =      0.1
					__lowercase        =      0.1
					__lowercase        =      config_megatron['''layernorm_epsilon''']  # 1e-5
					__lowercase        =      config_megatron['''init_method_std''']  # 0.02
					__lowercase        =      config_megatron['''apply_query_key_layer_scaling''']  # True
					__lowercase        =      True
					__lowercase        =      True

					# This identifies the 6.7B (7B) model which uses a different tokenizer
					if config_megatron["hidden_size"] == 40_96:
										__lowercase        =      1  # <|endoftext|>
										__lowercase        =      1  # <|endoftext|>
										__lowercase        =      0  # <unk>
					else:
										__lowercase        =      2  # <s>
										__lowercase        =      3  # <|endoftext|>
										__lowercase        =      0  # <pad>

					return config_hf





def 					__UpperCamelCase (     a__:			Dict      )					->      Union[str, Any]:




					"""simple docstring"""



					print(a__      )

					__lowercase        =      args.checkpoint_path
					__lowercase        =      args.save_path
					if isfile(a__      ):
										raise FileNotFoundError(f'''ERROR! could not find file {checkpoint_path}'''      )

					# Load the model.
					__lowercase        =      torch.load(a__			,						map_location='''cpu'''      )

					# Load the config.
					__lowercase        =      checkpoint['''hyper_parameters''']['''cfg''']
					__lowercase        =      GPTaConfig()
					__lowercase        =      copy_config(config_hf=a__			,						config_megatron=a__      )
					__lowercase        =      ['''GPT2LMHeadModel''']

					__lowercase        =      checkpoint['''state_dict''']

					# Convert.
					print('''Converting'''      )
					__lowercase        =      convert_megatron_checkpoint(a__			,						a__      )

					# Print the structure of converted state dict.
					if args.print_checkpoint_structure:
										recursive_print(a__			,						a__      )

					__lowercase        =      '''GPTSw3Tokenizer'''

					# Store the config to file.
					print('''Saving config'''      )
					config_hf.save_pretrained(a__      )

					# Store the state_dict to file.
					__lowercase        =      os.path.join(a__			,						'''pytorch_model.bin'''      )
					print(f'''Saving checkpoint to "{output_checkpoint_file}"'''      )
					torch.save(a__			,						a__      )


if __name__ == "__main__":
							a__			:						Union[str, Any]							=			argparse.ArgumentParser()
							parser.add_argument(
							    """--checkpoint_path""",
							    type=str,
							    required=True,
							    help="""e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000""",
							)
							parser.add_argument("""--save_path""", type=str, required=True, help="""e.g. /home/user/gpt-sw3/hf""")
							parser.add_argument("""--print-checkpoint-structure""", action="""store_true""")
							a__			:						Dict							=			parser.parse_args()
							main(_args)


