




import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig







def 		lowercase__ (		__A:						Optional[int]				,__A:						List[Any]				,__A:						Any=0					):





					'''simple docstring'''






					if name is None:
										__magic_name__				:						List[Any]     				=				None
					else:
										__magic_name__				:						Dict     				=				'''.''' * max(0				,spaces - 2					) + '''# {:''' + str(5_0 - spaces					) + '''s}'''
										__magic_name__				:						str     				=				fmt.format(__A					)

					# Print and recurse (if needed).
					if isinstance(__A				,__A					):
										if msg is not None:
															print(__A					)
										for k in val.keys():
															recursive_print(__A				,val[k]				,spaces + 2					)
					elif isinstance(__A				,torch.Tensor					):
										print(__A				,''':'''				,val.size()					)
					else:
										print(__A				,''':'''				,__A					)







def 		lowercase__ (		__A:						str				,__A:						List[str]				,__A:						Optional[int]				,__A:						str					):





					'''simple docstring'''






					__magic_name__				:						Optional[int]     				=				param.size()
					# other versions store [num_heads * num_splits * hidden_size, :]
					__magic_name__				:						Optional[Any]     				=				(num_heads, num_splits, hidden_size) + input_shape[1:]
					__magic_name__				:						Optional[Any]     				=				param.view(*__A					)
					__magic_name__				:						int     				=				param.transpose(0				,1					).contiguous()
					__magic_name__				:						Optional[int]     				=				param.view(*__A					)
					return param







def 		lowercase__ (		__A:						List[str]				,__A:						Union[str, Any]					):





					'''simple docstring'''






					__magic_name__				:						int     				=				config.n_positions
					__magic_name__				:						List[str]     				=				config.n_layer
					__magic_name__				:						int     				=				config.vocab_size
					__magic_name__				:						List[Any]     				=				config.n_head
					__magic_name__				:						Tuple     				=				config.n_embd // config.n_head

					__magic_name__				:						Optional[Any]     				=				sd_megatron['''model.language_model.embedding.word_embeddings.weight'''][:vocab_size, :]
					__magic_name__				:						Tuple     				=				{
					    '''transformer.wte.weight''': word_embeddings,
					    '''transformer.wpe.weight''': sd_megatron['''model.language_model.embedding.position_embeddings.weight'''],
					    '''transformer.ln_f.weight''': sd_megatron['''model.language_model.encoder.final_layernorm.weight'''],
					    '''transformer.ln_f.bias''': sd_megatron['''model.language_model.encoder.final_layernorm.bias'''],
					}

					__magic_name__				:						List[str]     				=				'''model.language_model.encoder.layers.'''
					for i in range(__A					):
										__magic_name__				:						str     				=				torch.tril(torch.ones((n_positions, n_positions)				,dtype=torch.bool					)					)
										__magic_name__				:						Dict     				=				causal_mask.view(1				,1				,__A				,__A					)
										__magic_name__				:						Tuple     				=				causal_mask
										__magic_name__				:						Optional[Any]     				=				torch.tensor(-1e4				,dtype=torch.bfloataa					)

										__magic_name__				:						List[Any]     				=				sd_megatron[F'''{pf}{i}.input_layernorm.weight''']
										__magic_name__				:						List[str]     				=				sd_megatron[F'''{pf}{i}.input_layernorm.bias''']

										__magic_name__				:						str     				=				sd_megatron[F'''{pf}{i}.self_attention.query_key_value.weight''']
										__magic_name__				:						Optional[Any]     				=				fix_query_key_value_ordering(__A				,3				,__A				,__A					)
										__magic_name__				:						str     				=				vala.transpose(0				,1					).contiguous()

										__magic_name__				:						int     				=				sd_megatron[F'''{pf}{i}.self_attention.query_key_value.bias''']
										__magic_name__				:						Any     				=				fix_query_key_value_ordering(__A				,3				,__A				,__A					)
										__magic_name__				:						Dict     				=				vala

										__magic_name__				:						str     				=				sd_megatron[F'''{pf}{i}.self_attention.dense.weight'''].transpose(
										    0				,1					)
										__magic_name__				:						Optional[int]     				=				sd_megatron[F'''{pf}{i}.self_attention.dense.bias''']
										__magic_name__				:						Optional[int]     				=				sd_megatron[F'''{pf}{i}.post_attention_layernorm.weight''']
										__magic_name__				:						List[Any]     				=				sd_megatron[F'''{pf}{i}.post_attention_layernorm.bias''']
										__magic_name__				:						List[Any]     				=				sd_megatron[F'''{pf}{i}.mlp.dense_h_to_4h.weight'''].transpose(0				,1					)
										__magic_name__				:						List[str]     				=				sd_megatron[F'''{pf}{i}.mlp.dense_h_to_4h.bias''']
										__magic_name__				:						Optional[Any]     				=				sd_megatron[F'''{pf}{i}.mlp.dense_4h_to_h.weight'''].transpose(
										    0				,1					)
										__magic_name__				:						Optional[Any]     				=				sd_megatron[F'''{pf}{i}.mlp.dense_4h_to_h.bias''']

					# For LM head, transformers' wants the matrix to weight embeddings.
					__magic_name__				:						Optional[int]     				=				word_embeddings

					return sd_hf







def 		lowercase__ (		__A:						Optional[Any]				,__A:						str					):





					'''simple docstring'''






					__magic_name__				:						Optional[Any]     				=				6_4_0_0_0
					__magic_name__				:						Dict     				=				config_megatron['''encoder_seq_length''']
					__magic_name__				:						str     				=				config_megatron['''hidden_size''']
					__magic_name__				:						Union[str, Any]     				=				config_megatron['''num_layers''']
					__magic_name__				:						Optional[Any]     				=				config_megatron['''num_attention_heads''']
					__magic_name__				:						Tuple     				=				config_megatron['''ffn_hidden_size''']
					__magic_name__				:						Any     				=				'''gelu'''
					__magic_name__				:						Optional[int]     				=				0.1
					__magic_name__				:						List[Any]     				=				0.1
					__magic_name__				:						str     				=				0.1
					__magic_name__				:						Optional[int]     				=				config_megatron['''layernorm_epsilon''']  # 1e-5
					__magic_name__				:						Optional[Any]     				=				config_megatron['''init_method_std''']  # 0.02
					__magic_name__				:						Optional[int]     				=				config_megatron['''apply_query_key_layer_scaling''']  # True
					__magic_name__				:						List[str]     				=				True
					__magic_name__				:						Dict     				=				True

					# This identifies the 6.7B (7B) model which uses a different tokenizer
					if config_megatron["hidden_size"] == 4_0_9_6:
										__magic_name__				:						Optional[Any]     				=				1  # <|endoftext|>
										__magic_name__				:						Union[str, Any]     				=				1  # <|endoftext|>
										__magic_name__				:						str     				=				0  # <unk>
					else:
										__magic_name__				:						Dict     				=				2  # <s>
										__magic_name__				:						List[Any]     				=				3  # <|endoftext|>
										__magic_name__				:						Optional[Any]     				=				0  # <pad>

					return config_hf







def 		lowercase__ (		__A:						Union[str, Any]					):





					'''simple docstring'''






					print(__A					)

					__magic_name__				:						Dict     				=				args.checkpoint_path
					__magic_name__				:						str     				=				args.save_path
					if isfile(__A					):
										raise FileNotFoundError(F'''ERROR! could not find file {checkpoint_path}'''					)

					# Load the model.
					__magic_name__				:						Optional[Any]     				=				torch.load(__A				,map_location='''cpu'''					)

					# Load the config.
					__magic_name__				:						Tuple     				=				checkpoint['''hyper_parameters''']['''cfg''']
					__magic_name__				:						List[str]     				=				GPTaConfig()
					__magic_name__				:						Any     				=				copy_config(config_hf=__A				,config_megatron=__A					)
					__magic_name__				:						Tuple     				=				['''GPT2LMHeadModel''']

					__magic_name__				:						Dict     				=				checkpoint['''state_dict''']

					# Convert.
					print('''Converting'''					)
					__magic_name__				:						Tuple     				=				convert_megatron_checkpoint(__A				,__A					)

					# Print the structure of converted state dict.
					if args.print_checkpoint_structure:
										recursive_print(__A				,__A					)

					__magic_name__				:						Tuple     				=				'''GPTSw3Tokenizer'''

					# Store the config to file.
					print('''Saving config'''					)
					config_hf.save_pretrained(__A					)

					# Store the state_dict to file.
					__magic_name__				:						List[Any]     				=				os.path.join(__A				,'''pytorch_model.bin'''					)
					print(F'''Saving checkpoint to "{output_checkpoint_file}"'''					)
					torch.save(__A				,__A					)


if __name__ == "__main__":
	__lowerCamelCase :    Dict      						=      argparse.ArgumentParser()
	parser.add_argument(
	    '''--checkpoint_path''',
	    type=str,
	    required=True,
	    help='''e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000''',
	)
	parser.add_argument('''--save_path''', type=str, required=True, help='''e.g. /home/user/gpt-sw3/hf''')
	parser.add_argument('''--print-checkpoint-structure''', action='''store_true''')
	__lowerCamelCase :    Tuple      						=      parser.parse_args()
	main(_args)









import math


class  lowerCamelCase     :






			'''simple docstring'''





			def __init__(			self      : Dict							,    lowerCamelCase_      : int							,    lowerCamelCase_      : int							,    lowerCamelCase_      : int							,    lowerCamelCase_      : Union[str, Any]	)    -> Optional[Any]:
								__magic_name__				:						Union[str, Any]     				=				x
								__magic_name__				:						Optional[int]     				=				y
								__magic_name__				:						List[str]     				=				step_size
								__magic_name__				:						Union[str, Any]     				=				function_to_optimize

			def 				UpperCAmelCase__							(			self      : int	)    -> int:
								return self.function(self.x							,    self.y	)

			def 				UpperCAmelCase__							(			self      : Dict	)    -> Any:
								__magic_name__				:						Optional[int]     				=				self.step_size
								return [
								    SearchProblem(lowerCamelCase_							,    lowerCamelCase_							,    lowerCamelCase_							,    self.function	)
								    for x, y in (
								        (self.x - step_size, self.y - step_size),
								        (self.x - step_size, self.y),
								        (self.x - step_size, self.y + step_size),
								        (self.x, self.y - step_size),
								        (self.x, self.y + step_size),
								        (self.x + step_size, self.y - step_size),
								        (self.x + step_size, self.y),
								        (self.x + step_size, self.y + step_size),
								    )
								]

			def __hash__(			self      : Dict	)    -> Tuple:
								return hash(str(self	)	)

			def __eq__(			self      : List[str]							,    lowerCamelCase_      : Dict	)    -> Union[str, Any]:
								if isinstance(lowerCamelCase_							,    lowerCamelCase_	):
													return hash(str(self	)	) == hash(str(lowerCamelCase_	)	)
								return False


			def __str__(			self      : Tuple	)    -> Tuple:
								return F'''x: {self.x} y: {self.y}'''







def 		lowercase__ (		__A:						int				,__A:						bool = True				,__A:						float = math.inf				,__A:						float = -math.inf				,__A:						float = math.inf				,__A:						float = -math.inf				,__A:						bool = False				,__A:						int = 1_0_0_0_0				,):





					'''simple docstring'''






					__magic_name__				:						Union[str, Any]     				=				search_prob
					__magic_name__				:						int     				=				[]  # list to store the current score at each iteration
					__magic_name__				:						Any     				=				0
					__magic_name__				:						Optional[Any]     				=				False
					__magic_name__				:						Dict     				=				set()
					while not solution_found and iterations < max_iter:
										visited.add(__A					)
										iterations += 1
										__magic_name__				:						int     				=				current_state.score()
										scores.append(__A					)
										__magic_name__				:						Union[str, Any]     				=				current_state.get_neighbors()
										__magic_name__				:						str     				=				-math.inf
										__magic_name__				:						Optional[Any]     				=				math.inf
										__magic_name__				:						Any     				=				None  # to hold the next best neighbor
										for neighbor in neighbors:
															if neighbor in visited:
																				continue  # do not want to visit the same state again
															if (
															    neighbor.x > max_x
															    or neighbor.x < min_x
															    or neighbor.y > max_y
															    or neighbor.y < min_y
															):
																				continue  # neighbor outside our bounds
															__magic_name__				:						Tuple     				=				neighbor.score() - current_score
															if find_max:  # finding max
																				# going to direction with greatest ascent
																				if change > max_change and change > 0:
																									__magic_name__				:						List[str]     				=				change
																									__magic_name__				:						List[str]     				=				neighbor
															else:  # finding min
																				# to direction with greatest descent
																				if change < min_change and change < 0:
																									__magic_name__				:						List[Any]     				=				change
																									__magic_name__				:						Optional[int]     				=				neighbor
										if next_state is not None:
															# we found at least one neighbor which improved the current state
															__magic_name__				:						Any     				=				next_state
										else:
															# since we have no neighbor that improves the solution we stop the search
															__magic_name__				:						List[str]     				=				True

					if visualization:
										from matplotlib import pyplot as plt

										plt.plot(range(__A					)				,__A					)
										plt.xlabel('''Iterations'''					)
										plt.ylabel('''Function values'''					)
										plt.show()

					return current_state


if __name__ == "__main__":
	import doctest

	doctest.testmod()







	def 		lowercase__ (		__A:						Optional[Any]				,__A:						str					):





						'''simple docstring'''






						return (x**2) + (y**2)

	# starting the problem with initial coordinates (3, 4)
	__lowerCamelCase :    Any      						=      SearchProblem(x=3, y=4, step_size=1, function_to_optimize=test_fa)
	__lowerCamelCase :    Any      						=      hill_climbing(prob, find_max=False)
	print(
	    '''The minimum score for f(x, y) = x^2 + y^2 found via hill climbing: '''
	    F"""{local_min.score()}"""
	)

	# starting the problem with initial coordinates (12, 47)
	__lowerCamelCase :    Optional[int]      						=      SearchProblem(x=12, y=47, step_size=1, function_to_optimize=test_fa)
	__lowerCamelCase :    Union[str, Any]      						=      hill_climbing(
	    prob, find_max=False, max_x=1_00, min_x=5, max_y=50, min_y=-5, visualization=True
	)
	print(
	    '''The minimum score for f(x, y) = x^2 + y^2 with the domain 100 > x > 5 '''
	    F"""and 50 > y > - 5 found via hill climbing: {local_min.score()}"""
	)







	def 		lowercase__ (		__A:						List[Any]				,__A:						int					):





						'''simple docstring'''






						return (3 * x**2) - (6 * y)

	__lowerCamelCase :    List[str]      						=      SearchProblem(x=3, y=4, step_size=1, function_to_optimize=test_fa)
	__lowerCamelCase :    Optional[int]      						=      hill_climbing(prob, find_max=True)
	print(
	    '''The maximum score for f(x, y) = x^2 + y^2 found via hill climbing: '''
	    F"""{local_min.score()}"""
	)



