





# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig



def 		A_				(     _lowerCAmelCase							,   _lowerCAmelCase							,   _lowerCAmelCase=0     )						-> Any:
 # Format the message.
 if name is None:
  UpperCamelCase	:					Dict       	=   None
 else:
  UpperCamelCase	:					Tuple       	=   "." * max(0							,   spaces - 2     ) + "# {:" + str(50 - spaces     ) + "s}"
  UpperCamelCase	:					List[str]       	=   fmt.format(_lowerCAmelCase     )

 # Print and recurse (if needed).
 if isinstance(_lowerCAmelCase							,   _lowerCAmelCase     ):
  if msg is not None:
   print(_lowerCAmelCase     )
  for k in val.keys():
   recursive_print(_lowerCAmelCase							,   val[k]							,   spaces + 2     )
 elif isinstance(_lowerCAmelCase							,   torch.Tensor     ):
  print(_lowerCAmelCase							,   ":"							,   val.size()     )
 else:
  print(_lowerCAmelCase							,   ":"							,   _lowerCAmelCase     )



def 		A_				(     _lowerCAmelCase							,   _lowerCAmelCase							,   _lowerCAmelCase							,   _lowerCAmelCase							,   _lowerCAmelCase     )						-> Dict:
 # Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
 # for compatibility with later versions of NVIDIA Megatron-LM.
 # The inverse operation is performed inside Megatron-LM to read checkpoints:
 # https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
 # If param is the weight tensor of the self-attention block, the returned tensor
 # will have to be transposed one more time to be read by HuggingFace GPT2.
 UpperCamelCase	:					str       	=   param.size()
 if checkpoint_version == 1.0:
  # version 1.0 stores [num_heads * hidden_size * num_splits, :]
  UpperCamelCase	:					List[str]       	=   (num_heads, hidden_size, num_splits) + input_shape[1:]
  UpperCamelCase	:					str       	=   param.view(*_lowerCAmelCase     )
  UpperCamelCase	:					Union[str, Any]       	=   param.transpose(0							,   2     )
  UpperCamelCase	:					List[Any]       	=   param.transpose(1							,   2     ).contiguous()
 elif checkpoint_version >= 2.0:
  # other versions store [num_heads * num_splits * hidden_size, :]
  UpperCamelCase	:					Optional[Any]       	=   (num_heads, num_splits, hidden_size) + input_shape[1:]
  UpperCamelCase	:					Dict       	=   param.view(*_lowerCAmelCase     )
  UpperCamelCase	:					Dict       	=   param.transpose(0							,   1     ).contiguous()
 UpperCamelCase	:					List[Any]       	=   param.view(*_lowerCAmelCase     )
 return param



def 		A_				(     _lowerCAmelCase							,   _lowerCAmelCase							,   _lowerCAmelCase     )						-> str:
 # The converted output model.
 UpperCamelCase	:					List[str]       	=   {}

 # old versions did not store training args
 UpperCamelCase	:					Optional[int]       	=   input_state_dict.get("args"							,   _lowerCAmelCase     )
 if ds_args is not None:
  # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
  # from pprint import pprint
  # pprint(vars(ds_args))

  UpperCamelCase	:					Optional[int]       	=   ds_args.padded_vocab_size
  UpperCamelCase	:					int       	=   ds_args.max_position_embeddings
  UpperCamelCase	:					Any       	=   ds_args.hidden_size
  UpperCamelCase	:					Dict       	=   ds_args.num_layers
  UpperCamelCase	:					str       	=   ds_args.num_attention_heads
  UpperCamelCase	:					Dict       	=   ds_args.ffn_hidden_size
  # pprint(config)

 # The number of heads.
 UpperCamelCase	:					Optional[Any]       	=   config.n_head
 # The hidden_size per head.
 UpperCamelCase	:					List[str]       	=   config.n_embd // config.n_head
 # Megatron-LM checkpoint version
 if "checkpoint_version" in input_state_dict.keys():
  UpperCamelCase	:					Optional[int]       	=   input_state_dict["checkpoint_version"]
 else:
  UpperCamelCase	:					Tuple       	=   0.0

 # The model.
 UpperCamelCase	:					Dict       	=   input_state_dict["model"]
 # The language model.
 UpperCamelCase	:					Any       	=   model["language_model"]
 # The embeddings.
 UpperCamelCase	:					Tuple       	=   lm["embedding"]

 # The word embeddings.
 UpperCamelCase	:					List[str]       	=   embeddings["word_embeddings"]["weight"]
 # Truncate the embedding table to vocab_size rows.
 UpperCamelCase	:					Dict       	=   word_embeddings[: config.vocab_size, :]
 UpperCamelCase	:					Optional[int]       	=   word_embeddings

 # The position embeddings.
 UpperCamelCase	:					List[str]       	=   embeddings["position_embeddings"]["weight"]
 # Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
 UpperCamelCase	:					Union[str, Any]       	=   pos_embeddings.size(0     )
 if n_positions != config.n_positions:
  raise ValueError(
      F"""pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don't match"""     )
 # Store the position embeddings.
 UpperCamelCase	:					List[Any]       	=   pos_embeddings

 # The transformer.
 UpperCamelCase	:					int       	=   lm["transformer"] if "transformer" in lm.keys() else lm["encoder"]

 # The regex to extract layer names.
 UpperCamelCase	:					List[Any]       	=   re.compile(r"layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"     )

 # The simple map of names for "automated" rules.
 UpperCamelCase	:					int       	=   {
     "attention.dense": ".attn.c_proj.",
     "self_attention.dense": ".attn.c_proj.",
     "mlp.dense_h_to_4h": ".mlp.c_fc.",
     "mlp.dense_4h_to_h": ".mlp.c_proj.",
 }

 # Extract the layers.
 for key, val in transformer.items():
  # Match the name.
  UpperCamelCase	:					List[str]       	=   layer_re.match(_lowerCAmelCase     )

  # Stop if that's not a layer
  if m is None:
   break

  # The index of the layer.
  UpperCamelCase	:					List[str]       	=   int(m.group(1     )     )
  # The name of the operation.
  UpperCamelCase	:					List[Any]       	=   m.group(2     )
  # Is it a weight or a bias?
  UpperCamelCase	:					str       	=   m.group(3     )

  # The name of the layer.
  UpperCamelCase	:					List[Any]       	=   F"""transformer.h.{layer_idx}"""

  # For layernorm(s), simply store the layer norm.
  if op_name.endswith("layernorm"     ):
   UpperCamelCase	:					Optional[int]       	=   "ln_1" if op_name.startswith("input"     ) else "ln_2"
   UpperCamelCase	:					str       	=   val

  # Transpose the QKV matrix.
  elif (
      op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
  ) and weight_or_bias == "weight":
   # Insert a tensor of 1x1xDxD bias.
   UpperCamelCase	:					str       	=   torch.tril(torch.ones((n_positions, n_positions)							,   dtype=torch.floataa     )     ).view(
       1							,   1							,   _lowerCAmelCase							,   _lowerCAmelCase     )
   UpperCamelCase	:					Optional[Any]       	=   causal_mask

   # Insert a "dummy" tensor for masked_bias.
   UpperCamelCase	:					Dict       	=   torch.tensor(-1e4							,   dtype=torch.floataa     )
   UpperCamelCase	:					Dict       	=   masked_bias

   UpperCamelCase	:					List[Any]       	=   fix_query_key_value_ordering(_lowerCAmelCase							,   _lowerCAmelCase							,   3							,   _lowerCAmelCase							,   _lowerCAmelCase     )
   # Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
   UpperCamelCase	:					Any       	=   out_val.transpose(0							,   1     ).contiguous()
   # Store.
   UpperCamelCase	:					int       	=   out_val

  # Transpose the bias.
  elif (
      op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
  ) and weight_or_bias == "bias":
   UpperCamelCase	:					Dict       	=   fix_query_key_value_ordering(_lowerCAmelCase							,   _lowerCAmelCase							,   3							,   _lowerCAmelCase							,   _lowerCAmelCase     )
   # Store. No change of shape.
   UpperCamelCase	:					Tuple       	=   out_val

  # Transpose the weights.
  elif weight_or_bias == "weight":
   UpperCamelCase	:					Tuple       	=   megatron_to_transformers[op_name]
   UpperCamelCase	:					Tuple       	=   val.transpose(0							,   1     )

  # Copy the bias.
  elif weight_or_bias == "bias":
   UpperCamelCase	:					int       	=   megatron_to_transformers[op_name]
   UpperCamelCase	:					str       	=   val

    # DEBUG.
 assert config.n_layer == layer_idx + 1

 # The final layernorm.
 UpperCamelCase	:					Optional[int]       	=   transformer["final_layernorm.weight"]
 UpperCamelCase	:					Optional[Any]       	=   transformer["final_layernorm.bias"]

 # For LM head, transformers' wants the matrix to weight embeddings.
 UpperCamelCase	:					List[Any]       	=   word_embeddings

 # It should be done!
 return output_state_dict



def 		A_				(     )						-> List[str]:
 # Create the argument parser.
 UpperCamelCase	:					str       	=   argparse.ArgumentParser()
 parser.add_argument("--print-checkpoint-structure"							,   action="store_true"     )
 parser.add_argument(
     "path_to_checkpoint"							,   type=_lowerCAmelCase							,   help="Path to the checkpoint file (.zip archive or direct .pt file)"							,   )
 parser.add_argument(
     "--config_file"							,   default=""							,   type=_lowerCAmelCase							,   help="An optional config json file describing the pre-trained model."							,   )
 UpperCamelCase	:					List[str]       	=   parser.parse_args()

 # Extract the basename.
 UpperCamelCase	:					List[Any]       	=   os.path.dirname(args.path_to_checkpoint     )

 # Load the model.
 # the .zip is very optional, let's keep it for backward compatibility
 print(F"""Extracting PyTorch state dictionary from {args.path_to_checkpoint}"""     )
 if args.path_to_checkpoint.endswith(".zip"     ):
  with zipfile.ZipFile(args.path_to_checkpoint							,   "r"     ) as checkpoint:
   with checkpoint.open("release/mp_rank_00/model_optim_rng.pt"     ) as pytorch_dict:
    UpperCamelCase	:					int       	=   torch.load(_lowerCAmelCase							,   map_location="cpu"     )
 else:
  UpperCamelCase	:					List[Any]       	=   torch.load(args.path_to_checkpoint							,   map_location="cpu"     )

 UpperCamelCase	:					Optional[Any]       	=   input_state_dict.get("args"							,   _lowerCAmelCase     )

 # Read the config, or default to the model released by NVIDIA.
 if args.config_file == "":
  if ds_args is not None:
   if ds_args.bias_gelu_fusion:
    UpperCamelCase	:					Dict       	=   "gelu_fast"
   elif ds_args.openai_gelu:
    UpperCamelCase	:					str       	=   "gelu_new"
   else:
    UpperCamelCase	:					Tuple       	=   "gelu"
  else:
   # in the very early days this used to be "gelu_new"
   UpperCamelCase	:					Tuple       	=   "gelu_new"

  # Spell out all parameters in case the defaults change.
  UpperCamelCase	:					List[Any]       	=   GPTaConfig(
      vocab_size=5_0257							,   n_positions=1024							,   n_embd=1024							,   n_layer=24							,   n_head=16							,   n_inner=4096							,   activation_function=_lowerCAmelCase							,   resid_pdrop=0.1							,   embd_pdrop=0.1							,   attn_pdrop=0.1							,   layer_norm_epsilon=1e-5							,   initializer_range=0.02							,   summary_type="cls_index"							,   summary_use_proj=_lowerCAmelCase							,   summary_activation=_lowerCAmelCase							,   summary_proj_to_labels=_lowerCAmelCase							,   summary_first_dropout=0.1							,   scale_attn_weights=_lowerCAmelCase							,   use_cache=_lowerCAmelCase							,   bos_token_id=5_0256							,   eos_token_id=5_0256							,   )
 else:
  UpperCamelCase	:					int       	=   GPTaConfig.from_json_file(args.config_file     )

 UpperCamelCase	:					str       	=   ["GPT2LMHeadModel"]

 # Convert.
 print("Converting"     )
 UpperCamelCase	:					Dict       	=   convert_megatron_checkpoint(_lowerCAmelCase							,   _lowerCAmelCase							,   _lowerCAmelCase     )

 # Print the structure of converted state dict.
 if args.print_checkpoint_structure:
  recursive_print(_lowerCAmelCase							,   _lowerCAmelCase     )

 # Add tokenizer class info to config
 # see https://github.com/huggingface/transformers/issues/13906)
 if ds_args is not None:
  UpperCamelCase	:					Union[str, Any]       	=   ds_args.tokenizer_type
  if tokenizer_type == "GPT2BPETokenizer":
   UpperCamelCase	:					Optional[int]       	=   "gpt2"
  elif tokenizer_type == "PretrainedFromHF":
   UpperCamelCase	:					Any       	=   ds_args.tokenizer_name_or_path
  else:
   raise ValueError(F"""Unrecognized tokenizer_type {tokenizer_type}"""     )
 else:
  UpperCamelCase	:					Optional[int]       	=   "gpt2"

 UpperCamelCase	:					Optional[Any]       	=   AutoTokenizer.from_pretrained(_lowerCAmelCase     )
 UpperCamelCase	:					Tuple       	=   type(_lowerCAmelCase     ).__name__
 UpperCamelCase	:					Optional[int]       	=   tokenizer_class

 # Store the config to file.
 print("Saving config"     )
 config.save_pretrained(_lowerCAmelCase     )

 # Save tokenizer based on args
 print(F"""Adding {tokenizer_class} tokenizer files"""     )
 tokenizer.save_pretrained(_lowerCAmelCase     )

 # Store the state_dict to file.
 UpperCamelCase	:					Optional[Any]       	=   os.path.join(_lowerCAmelCase							,   "pytorch_model.bin"     )
 print(F"""Saving checkpoint to \"{output_checkpoint_file}\""""     )
 torch.save(_lowerCAmelCase							,   _lowerCAmelCase     )


####################################################################################################

if __name__ == "__main__":
    main()

####################################################################################################



"""simple docstring"""





import unittest

from transformers import load_tool

from .test_tools_common import ToolTesterMixin




class 					_lowerCAmelCase  (	unittest.TestCase     ,lowercase   ):







 """simple docstring"""







 def 					_lowercase (			self							:			List[Any]	):
    __lowercase								=						load_tool("text-classification"	)
    self.tool.setup()
    __lowercase								=						load_tool("text-classification",					remote=UpperCAmelCase__	)


 def 					_lowercase (			self							:			str	):
    __lowercase								=						self.tool("That's quite cool",					["positive", "negative"]	)
    self.assertEqual(UpperCAmelCase__,					"positive"	)


 def 					_lowercase (			self							:			str	):
    __lowercase								=						self.remote_tool("That's quite cool",					["positive", "negative"]	)
    self.assertEqual(UpperCAmelCase__,					"positive"	)


 def 					_lowercase (			self							:			List[str]	):
    __lowercase								=						self.tool(text="That's quite cool",					labels=["positive", "negative"]	)
    self.assertEqual(UpperCAmelCase__,					"positive"	)



 def 					_lowercase (			self							:			Tuple	):
    __lowercase								=						self.remote_tool(text="That's quite cool",					labels=["positive", "negative"]	)
    self.assertEqual(UpperCAmelCase__,					"positive"	)




