
'''simple docstring'''







# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig







def 				SCREAMING_SNAKE_CASE__	(   __A			,				__A			,				__A=0	)       ->	List[str]:
	# Format the message.
	if name is None:
		_snake_case   						=		None
	else:
		_snake_case   						=		'.' * max(0			,				spaces - 2	) + '# {:' + str(50 - spaces	) + 's}'
		_snake_case   						=		fmt.format(__A	)

	# Print and recurse (if needed).
	if isinstance(__A			,				__A	):
		if msg is not None:
			print(__A	)
		for k in val.keys():
			recursive_print(__A			,				val[k]			,				spaces + 2	)
	elif isinstance(__A			,				torch.Tensor	):
		print(__A			,				':'			,				val.size()	)
	else:
		print(__A			,				':'			,				__A	)







def 				SCREAMING_SNAKE_CASE__	(   __A			,				__A			,				__A			,				__A			,				__A	)       ->	Optional[Any]:
	# Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
	# for compatibility with later versions of NVIDIA Megatron-LM.
	# The inverse operation is performed inside Megatron-LM to read checkpoints:
	# https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
	# If param is the weight tensor of the self-attention block, the returned tensor
	# will have to be transposed one more time to be read by HuggingFace GPT2.
	_snake_case   						=		param.size()
	if checkpoint_version == 1.0:
		# version 1.0 stores [num_heads * hidden_size * num_splits, :]
		_snake_case   						=		(num_heads, hidden_size, num_splits) + input_shape[1:]
		_snake_case   						=		param.view(*__A	)
		_snake_case   						=		param.transpose(0			,				2	)
		_snake_case   						=		param.transpose(1			,				2	).contiguous()
	elif checkpoint_version >= 2.0:
		# other versions store [num_heads * num_splits * hidden_size, :]
		_snake_case   						=		(num_heads, num_splits, hidden_size) + input_shape[1:]
		_snake_case   						=		param.view(*__A	)
		_snake_case   						=		param.transpose(0			,				1	).contiguous()
	_snake_case   						=		param.view(*__A	)
	return param







def 				SCREAMING_SNAKE_CASE__	(   __A			,				__A			,				__A	)       ->	Union[str, Any]:
	# The converted output model.
	_snake_case   						=		{}

	# old versions did not store training args
	_snake_case   						=		input_state_dict.get('args'			,				__A	)
	if ds_args is not None:
		# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
		# from pprint import pprint
		# pprint(vars(ds_args))

		_snake_case   						=		ds_args.padded_vocab_size
		_snake_case   						=		ds_args.max_position_embeddings
		_snake_case   						=		ds_args.hidden_size
		_snake_case   						=		ds_args.num_layers
		_snake_case   						=		ds_args.num_attention_heads
		_snake_case   						=		ds_args.ffn_hidden_size
		# pprint(config)

	# The number of heads.
	_snake_case   						=		config.n_head
	# The hidden_size per head.
	_snake_case   						=		config.n_embd // config.n_head
	# Megatron-LM checkpoint version
	if "checkpoint_version" in input_state_dict.keys():
		_snake_case   						=		input_state_dict['checkpoint_version']
	else:
		_snake_case   						=		0.0

	# The model.
	_snake_case   						=		input_state_dict['model']
	# The language model.
	_snake_case   						=		model['language_model']
	# The embeddings.
	_snake_case   						=		lm['embedding']

	# The word embeddings.
	_snake_case   						=		embeddings['word_embeddings']['weight']
	# Truncate the embedding table to vocab_size rows.
	_snake_case   						=		word_embeddings[: config.vocab_size, :]
	_snake_case   						=		word_embeddings

	# The position embeddings.
	_snake_case   						=		embeddings['position_embeddings']['weight']
	# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
	_snake_case   						=		pos_embeddings.size(0	)
	if n_positions != config.n_positions:
		raise ValueError(
		    F'pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match'	)
	# Store the position embeddings.
	_snake_case   						=		pos_embeddings

	# The transformer.
	_snake_case   						=		lm['transformer'] if 'transformer' in lm.keys() else lm['encoder']

	# The regex to extract layer names.
	_snake_case   						=		re.compile(r'layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'	)

	# The simple map of names for "automated" rules.
	_snake_case   						=		{
	    'attention.dense': '.attn.c_proj.',
	    'self_attention.dense': '.attn.c_proj.',
	    'mlp.dense_h_to_4h': '.mlp.c_fc.',
	    'mlp.dense_4h_to_h': '.mlp.c_proj.',
	}

	# Extract the layers.
	for key, val in transformer.items():
		# Match the name.
		_snake_case   						=		layer_re.match(__A	)

		# Stop if that's not a layer
		if m is None:
			break

		# The index of the layer.
		_snake_case   						=		int(m.group(1	)	)
		# The name of the operation.
		_snake_case   						=		m.group(2	)
		# Is it a weight or a bias?
		_snake_case   						=		m.group(3	)

		# The name of the layer.
		_snake_case   						=		F'transformer.h.{layer_idx}'

		# For layernorm(s), simply store the layer norm.
		if op_name.endswith('layernorm'	):
			_snake_case   						=		'ln_1' if op_name.startswith('input'	) else 'ln_2'
			_snake_case   						=		val

		# Transpose the QKV matrix.
		elif (
		    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
		) and weight_or_bias == "weight":
			# Insert a tensor of 1x1xDxD bias.
			_snake_case   						=		torch.tril(torch.ones((n_positions, n_positions)			,				dtype=torch.floataa	)	).view(
			    1			,				1			,				__A			,				__A	)
			_snake_case   						=		causal_mask

			# Insert a "dummy" tensor for masked_bias.
			_snake_case   						=		torch.tensor(-1e4			,				dtype=torch.floataa	)
			_snake_case   						=		masked_bias

			_snake_case   						=		fix_query_key_value_ordering(__A			,				__A			,				3			,				__A			,				__A	)
			# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
			_snake_case   						=		out_val.transpose(0			,				1	).contiguous()
			# Store.
			_snake_case   						=		out_val

		# Transpose the bias.
		elif (
		    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
		) and weight_or_bias == "bias":
			_snake_case   						=		fix_query_key_value_ordering(__A			,				__A			,				3			,				__A			,				__A	)
			# Store. No change of shape.
			_snake_case   						=		out_val

		# Transpose the weights.
		elif weight_or_bias == "weight":
			_snake_case   						=		megatron_to_transformers[op_name]
			_snake_case   						=		val.transpose(0			,				1	)

		# Copy the bias.
		elif weight_or_bias == "bias":
			_snake_case   						=		megatron_to_transformers[op_name]
			_snake_case   						=		val

    # DEBUG.
	assert config.n_layer == layer_idx + 1

	# The final layernorm.
	_snake_case   						=		transformer['final_layernorm.weight']
	_snake_case   						=		transformer['final_layernorm.bias']

	# For LM head, transformers' wants the matrix to weight embeddings.
	_snake_case   						=		word_embeddings

	# It should be done!
	return output_state_dict







def 				SCREAMING_SNAKE_CASE__	(   )       ->	Tuple:
	# Create the argument parser.
	_snake_case   						=		argparse.ArgumentParser()
	parser.add_argument('--print-checkpoint-structure'			,				action='store_true'	)
	parser.add_argument(
	    'path_to_checkpoint'			,				type=__A			,				help='Path to the checkpoint file (.zip archive or direct .pt file)'			,				)
	parser.add_argument(
	    '--config_file'			,				default=''			,				type=__A			,				help='An optional config json file describing the pre-trained model.'			,				)
	_snake_case   						=		parser.parse_args()

	# Extract the basename.
	_snake_case   						=		os.path.dirname(args.path_to_checkpoint	)

	# Load the model.
	# the .zip is very optional, let's keep it for backward compatibility
	print(F'Extracting PyTorch state dictionary from {args.path_to_checkpoint}'	)
	if args.path_to_checkpoint.endswith('.zip'	):
		with zipfile.ZipFile(args.path_to_checkpoint			,				'r'	) as checkpoint:
			with checkpoint.open('release/mp_rank_00/model_optim_rng.pt'	) as pytorch_dict:
				_snake_case   						=		torch.load(__A			,				map_location='cpu'	)
	else:
		_snake_case   						=		torch.load(args.path_to_checkpoint			,				map_location='cpu'	)

	_snake_case   						=		input_state_dict.get('args'			,				__A	)

	# Read the config, or default to the model released by NVIDIA.
	if args.config_file == "":
		if ds_args is not None:
			if ds_args.bias_gelu_fusion:
				_snake_case   						=		'gelu_fast'
			elif ds_args.openai_gelu:
				_snake_case   						=		'gelu_new'
			else:
				_snake_case   						=		'gelu'
		else:
			# in the very early days this used to be "gelu_new"
			_snake_case   						=		'gelu_new'

		# Spell out all parameters in case the defaults change.
		_snake_case   						=		GPTaConfig(
		    vocab_size=50_257			,				n_positions=1_024			,				n_embd=1_024			,				n_layer=24			,				n_head=16			,				n_inner=4_096			,				activation_function=__A			,				resid_pdrop=0.1			,				embd_pdrop=0.1			,				attn_pdrop=0.1			,				layer_norm_epsilon=1e-5			,				initializer_range=0.0_2			,				summary_type='cls_index'			,				summary_use_proj=__A			,				summary_activation=__A			,				summary_proj_to_labels=__A			,				summary_first_dropout=0.1			,				scale_attn_weights=__A			,				use_cache=__A			,				bos_token_id=50_256			,				eos_token_id=50_256			,				)
	else:
		_snake_case   						=		GPTaConfig.from_json_file(args.config_file	)

	_snake_case   						=		['GPT2LMHeadModel']

	# Convert.
	print('Converting'	)
	_snake_case   						=		convert_megatron_checkpoint(__A			,				__A			,				__A	)

	# Print the structure of converted state dict.
	if args.print_checkpoint_structure:
		recursive_print(__A			,				__A	)

	# Add tokenizer class info to config
	# see https://github.com/huggingface/transformers/issues/13906)
	if ds_args is not None:
		_snake_case   						=		ds_args.tokenizer_type
		if tokenizer_type == "GPT2BPETokenizer":
			_snake_case   						=		'gpt2'
		elif tokenizer_type == "PretrainedFromHF":
			_snake_case   						=		ds_args.tokenizer_name_or_path
		else:
			raise ValueError(F'Unrecognized tokenizer_type {tokenizer_type}'	)
	else:
		_snake_case   						=		'gpt2'

	_snake_case   						=		AutoTokenizer.from_pretrained(__A	)
	_snake_case   						=		type(__A	).__name__
	_snake_case   						=		tokenizer_class

	# Store the config to file.
	print('Saving config'	)
	config.save_pretrained(__A	)

	# Save tokenizer based on args
	print(F'Adding {tokenizer_class} tokenizer files'	)
	tokenizer.save_pretrained(__A	)

	# Store the state_dict to file.
	_snake_case   						=		os.path.join(__A			,				'pytorch_model.bin'	)
	print(F'Saving checkpoint to "{output_checkpoint_file}"'	)
	torch.save(__A			,				__A	)


####################################################################################################

if __name__ == "__main__":
					main()

####################################################################################################



def 				a						(							snake_case__:					int = 100					):


				'''simple docstring'''
				lowercase_       =					(n * (n + 1) // 2) ** 2
				lowercase_       =					n * (n + 1) * (2 * n + 1) // 6
				return sum_cubes - sum_squares


if __name__ == "__main__":
						print(f"{solution() = }")




