






import argparse
import os
import re
import zipfile

import torch

from transformers import MegatronBertConfig







def 						_UpperCamelCase (     UpperCamelCase_							:       List[Any]     ,							UpperCamelCase_							:       Optional[int]     ,							UpperCamelCase_							:       str=0		)	-> int:

						"""simple docstring"""



						if name is None:
												lowerCAmelCase__         =    None
						else:
												lowerCAmelCase__         =    '''.''' * max(0     ,							spaces - 2		) + '''# {:''' + str(50 - spaces		) + '''s}'''
												lowerCAmelCase__         =    fmt.format(_lowercase		)

						# Print and recurse (if needed).
						if isinstance(_lowercase     ,							_lowercase		):
												if msg is not None:
																		print(_lowercase		)
												for k in val.keys():
																		recursive_print(_lowercase     ,							val[k]     ,							spaces + 2		)
						elif isinstance(_lowercase     ,							torch.Tensor		):
												print(_lowercase     ,							':'     ,							val.size()		)
						else:
												print(_lowercase     ,							':'     ,							_lowercase		)







def 						_UpperCamelCase (     UpperCamelCase_							:       List[Any]     ,							UpperCamelCase_							:       Union[str, Any]     ,							UpperCamelCase_							:       int     ,							UpperCamelCase_							:       Optional[Any]     ,							UpperCamelCase_							:       Dict		)	-> str:

						"""simple docstring"""



						lowerCAmelCase__         =    param.size()
						if checkpoint_version == 1.0:
												# version 1.0 stores [num_heads * hidden_size * num_splits, :]
												lowerCAmelCase__         =    (num_heads, hidden_size, num_splits) + input_shape[1:]
												lowerCAmelCase__         =    param.view(*_lowercase		)
												lowerCAmelCase__         =    param.transpose(0     ,							2		)
												lowerCAmelCase__         =    param.transpose(1     ,							2		).contiguous()
						elif checkpoint_version >= 2.0:
												# other versions store [num_heads * num_splits * hidden_size, :]
												lowerCAmelCase__         =    (num_heads, num_splits, hidden_size) + input_shape[1:]
												lowerCAmelCase__         =    param.view(*_lowercase		)
												lowerCAmelCase__         =    param.transpose(0     ,							1		).contiguous()
						lowerCAmelCase__         =    param.view(*_lowercase		)
						return param







def 						_UpperCamelCase (     UpperCamelCase_							:       Tuple     ,							UpperCamelCase_							:       Tuple     ,							UpperCamelCase_							:       Any		)	-> str:

						"""simple docstring"""



						lowerCAmelCase__         =    {}

						# old versions did not store training args
						lowerCAmelCase__         =    input_state_dict.get('args'     ,							_lowercase		)
						if ds_args is not None:
												# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
												# from pprint import pprint
												# pprint(vars(ds_args))

												lowerCAmelCase__         =    ds_args.tokenizer_type
												lowerCAmelCase__         =    ds_args.padded_vocab_size
												lowerCAmelCase__         =    ds_args.max_position_embeddings
												lowerCAmelCase__         =    ds_args.hidden_size
												lowerCAmelCase__         =    ds_args.num_layers
												lowerCAmelCase__         =    ds_args.num_attention_heads
												lowerCAmelCase__         =    ds_args.ffn_hidden_size if '''ffn_hidden_size''' in ds_args else 4 * ds_args.hidden_size
												# pprint(config)

						# The number of heads.
						lowerCAmelCase__         =    config.num_attention_heads
						# The hidden_size per head.
						lowerCAmelCase__         =    config.hidden_size // heads
						# Megatron-LM checkpoint version
						if "checkpoint_version" in input_state_dict.keys():
												lowerCAmelCase__         =    input_state_dict['''checkpoint_version''']
						else:
												lowerCAmelCase__         =    0.0

						# The model.
						lowerCAmelCase__         =    input_state_dict['''model''']
						# The language model.
						lowerCAmelCase__         =    model['''language_model''']
						# The embeddings.
						lowerCAmelCase__         =    lm['''embedding''']

						# The word embeddings.
						lowerCAmelCase__         =    embeddings['''word_embeddings''']['''weight''']
						# Truncate the embedding table to vocab_size rows.
						lowerCAmelCase__         =    word_embeddings[: config.vocab_size, :]
						# Store the word embeddings.
						lowerCAmelCase__         =    word_embeddings

						# The position embeddings.
						lowerCAmelCase__         =    embeddings['''position_embeddings''']['''weight''']
						assert pos_embeddings.size(0		) == config.max_position_embeddings and pos_embeddings.size(1		) == config.hidden_size
						# Store the position embeddings.
						lowerCAmelCase__         =    pos_embeddings

						# The token-type embeddings.
						lowerCAmelCase__         =    embeddings['''tokentype_embeddings''']['''weight''']
						# Store the position embeddings.
						lowerCAmelCase__         =    tokentype_embeddings

						# The transformer.
						lowerCAmelCase__         =    lm['''transformer'''] if '''transformer''' in lm.keys() else lm['''encoder''']

						# The regex to extract layer names.
						lowerCAmelCase__         =    re.compile(r'layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'		)

						# The simple map of names for "automated" rules.
						lowerCAmelCase__         =    {
						    '''attention.dense''': '''.attention.output.dense.''',
						    '''self_attention.dense''': '''.attention.output.dense.''',
						    '''mlp.dense_h_to_4h''': '''.intermediate.dense.''',
						    '''mlp.dense_4h_to_h''': '''.output.dense.''',
						}

						# Keep track of the attention/query/value tensor.
						lowerCAmelCase__         =    None

						# Extract the layers.
						for key, val in transformer.items():
												# Match the name.
												lowerCAmelCase__         =    layer_re.match(_lowercase		)

												# Stop if that's not a layer
												if m is None:
																		break

												# The index of the layer.
												lowerCAmelCase__         =    int(m.group(1		)		)
												# The name of the operation.
												lowerCAmelCase__         =    m.group(2		)
												# Is it a weight or a bias?
												lowerCAmelCase__         =    m.group(3		)

												# The name of the layer.
												lowerCAmelCase__         =    F"bert.encoder.layer.{layer_idx}"

												# For layernorm(s), simply store the layer norm.
												if op_name.endswith('layernorm'		):
																		lowerCAmelCase__         =    '''attention.ln''' if op_name.startswith('input'		) else '''ln'''
																		lowerCAmelCase__         =    val

												# Transpose the QKV matrix.
												elif (
												    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
												) and weight_or_bias == "weight":
																		# Make sure the QKV pointer is nil.
																		assert attention_qkv_weight is None, ""

																		lowerCAmelCase__         =    fix_query_key_value_ordering(_lowercase     ,							_lowercase     ,							3     ,							_lowercase     ,							_lowercase		)
																		# Store the tensor as we need the bias as well to interleave QKV and biases.
																		lowerCAmelCase__         =    out_val

												# Transpose the bias.
												elif (
												    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
												) and weight_or_bias == "bias":
																		# Make sure we read the weight tensor.
																		assert attention_qkv_weight is not None, ""

																		# Split the QKV matrix into Q, K and V. Megatron stores Q,K,V interleaved.
																		lowerCAmelCase__         =    attention_qkv_weight[0 * config.hidden_size : 1 * config.hidden_size, :]
																		lowerCAmelCase__         =    attention_qkv_weight[1 * config.hidden_size : 2 * config.hidden_size, :]
																		lowerCAmelCase__         =    attention_qkv_weight[2 * config.hidden_size : 3 * config.hidden_size, :]

																		lowerCAmelCase__         =    fix_query_key_value_ordering(_lowercase     ,							_lowercase     ,							3     ,							_lowercase     ,							_lowercase		)
																		# Split the bias.
																		lowerCAmelCase__         =    out_val[0 * config.hidden_size : 1 * config.hidden_size]
																		lowerCAmelCase__         =    out_val[1 * config.hidden_size : 2 * config.hidden_size]
																		lowerCAmelCase__         =    out_val[2 * config.hidden_size : 3 * config.hidden_size]

																		# Store.
																		lowerCAmelCase__         =    q
																		lowerCAmelCase__         =    q_bias
																		lowerCAmelCase__         =    k
																		lowerCAmelCase__         =    k_bias
																		lowerCAmelCase__         =    v
																		lowerCAmelCase__         =    v_bias

																		# Clear the stored tensor.
																		lowerCAmelCase__         =    None

												# Copy weights and biases as is.
												elif weight_or_bias in ["weight", "bias"]:
																		lowerCAmelCase__         =    megatron_to_transformers[op_name]
																		lowerCAmelCase__         =    val

    # The final layernorm.
						lowerCAmelCase__         =    transformer['''final_layernorm.weight''']
						lowerCAmelCase__         =    transformer['''final_layernorm.bias''']

						# The pooler.
						lowerCAmelCase__         =    lm['''pooler''']

						# Store the matrix and the bias.
						lowerCAmelCase__         =    pooler['''dense.weight''']
						lowerCAmelCase__         =    pooler['''dense.bias''']

						# The LM head from Megatron (for RACE).
						lowerCAmelCase__         =    model['''lm_head''']

						# The transform matrix.
						lowerCAmelCase__         =    lm_head['''dense.weight''']
						lowerCAmelCase__         =    lm_head['''dense.bias''']

						# The transform LN.
						lowerCAmelCase__         =    lm_head['''layernorm.weight''']
						lowerCAmelCase__         =    lm_head['''layernorm.bias''']

						# For the decoder, we replicate the weights.
						lowerCAmelCase__         =    word_embeddings
						lowerCAmelCase__         =    lm_head['''bias''']

						# The classifier from Megatron (for MLNI).
						lowerCAmelCase__         =    model['''binary_head''']

						# Store the classifier.
						lowerCAmelCase__         =    binary_head['''weight''']
						lowerCAmelCase__         =    binary_head['''bias''']

						# It should be done!
						return output_state_dict







def 						_UpperCamelCase (     )	-> int:

						"""simple docstring"""



						lowerCAmelCase__         =    argparse.ArgumentParser()
						parser.add_argument('--print-checkpoint-structure'     ,							action='store_true'		)
						parser.add_argument('path_to_checkpoint'     ,							type=_lowercase     ,							help='Path to the ZIP file containing the checkpoint'		)
						parser.add_argument(
						    '--config_file'     ,							default=''     ,							type=_lowercase     ,							help='An optional config json file describing the pre-trained model.'     ,							)
						lowerCAmelCase__         =    parser.parse_args()

						# Extract the basename.
						lowerCAmelCase__         =    os.path.dirname(args.path_to_checkpoint		)

						# Load the model.
						# the .zip is very optional, let's keep it for backward compatibility
						print(F"Extracting PyTorch state dictionary from \"{args.path_to_checkpoint}\""		)
						if args.path_to_checkpoint.endswith('.zip'		):
												with zipfile.ZipFile(args.path_to_checkpoint     ,							'r'		) as checkpoint:
																		with checkpoint.open('release/mp_rank_00/model_optim_rng.pt'		) as pytorch_dict:
																								lowerCAmelCase__         =    torch.load(_lowercase     ,							map_location='cpu'		)
						else:
												lowerCAmelCase__         =    torch.load(args.path_to_checkpoint     ,							map_location='cpu'		)

						if args.config_file == "":
												# Default config of megatron-bert 345m
												lowerCAmelCase__         =    MegatronBertConfig()

												# different megatron-bert-*-345m models have different vocab sizes, so override the default
												# config (which is for megatron-bert-cased-345m) with the actual vocab dimension
												lowerCAmelCase__         =    input_state_dict['''model''']['''lm_head''']['''bias'''].numel()
						else:
												lowerCAmelCase__         =    MegatronBertConfig.from_json_file(args.config_file		)

						# Convert.
						print('Converting'		)
						lowerCAmelCase__         =    convert_megatron_checkpoint(_lowercase     ,							_lowercase     ,							_lowercase		)

						# Print the structure of converted state dict.
						if args.print_checkpoint_structure:
												recursive_print(_lowercase     ,							_lowercase		)

						# Store the config to file.
						print('Saving config'		)
						config.save_pretrained(_lowercase		)

						# Store the state_dict to file.
						lowerCAmelCase__         =    os.path.join(_lowercase     ,							'pytorch_model.bin'		)
						print(F"Saving checkpoint to \"{output_checkpoint_file}\""		)
						torch.save(_lowercase     ,							_lowercase		)


####################################################################################################

if __name__ == "__main__":
					main()

####################################################################################################









'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__UpperCAmelCase     :Union[str, Any]												=  {
    "configuration_mpnet": ["MPNET_PRETRAINED_CONFIG_ARCHIVE_MAP", "MPNetConfig"],
    "tokenization_mpnet": ["MPNetTokenizer"],
}

try:
      if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __UpperCAmelCase     :Optional[int]												=  ["MPNetTokenizerFast"]

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __UpperCAmelCase     :Union[str, Any]												=  [
          "MPNET_PRETRAINED_MODEL_ARCHIVE_LIST",
          "MPNetForMaskedLM",
          "MPNetForMultipleChoice",
          "MPNetForQuestionAnswering",
          "MPNetForSequenceClassification",
          "MPNetForTokenClassification",
          "MPNetLayer",
          "MPNetModel",
          "MPNetPreTrainedModel",
      ]

try:
      if not is_tf_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __UpperCAmelCase     :Any												=  [
          "TF_MPNET_PRETRAINED_MODEL_ARCHIVE_LIST",
          "TFMPNetEmbeddings",
          "TFMPNetForMaskedLM",
          "TFMPNetForMultipleChoice",
          "TFMPNetForQuestionAnswering",
          "TFMPNetForSequenceClassification",
          "TFMPNetForTokenClassification",
          "TFMPNetMainLayer",
          "TFMPNetModel",
          "TFMPNetPreTrainedModel",
      ]


if TYPE_CHECKING:
      from .configuration_mpnet import MPNET_PRETRAINED_CONFIG_ARCHIVE_MAP, MPNetConfig
      from .tokenization_mpnet import MPNetTokenizer

      try:
            if not is_tokenizers_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .tokenization_mpnet_fast import MPNetTokenizerFast

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_mpnet import (
                MPNET_PRETRAINED_MODEL_ARCHIVE_LIST,
                MPNetForMaskedLM,
                MPNetForMultipleChoice,
                MPNetForQuestionAnswering,
                MPNetForSequenceClassification,
                MPNetForTokenClassification,
                MPNetLayer,
                MPNetModel,
                MPNetPreTrainedModel,
            )

      try:
            if not is_tf_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_tf_mpnet import (
                TF_MPNET_PRETRAINED_MODEL_ARCHIVE_LIST,
                TFMPNetEmbeddings,
                TFMPNetForMaskedLM,
                TFMPNetForMultipleChoice,
                TFMPNetForQuestionAnswering,
                TFMPNetForSequenceClassification,
                TFMPNetForTokenClassification,
                TFMPNetMainLayer,
                TFMPNetModel,
                TFMPNetPreTrainedModel,
            )

else:
      import sys

      __UpperCAmelCase     :Optional[Any]												=  _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)