




import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig
def 							__lowerCAmelCase			(UpperCAmelCase_ ,     UpperCAmelCase_ ,     UpperCAmelCase_=0		):
    # Format the message.
    if name is None:
        a__  			=							None
    else:
        a__  			=							""".""" * max(0 ,     spaces - 2		) + """# {:""" + str(50 - spaces		) + """s}"""
        a__  			=							fmt.format(UpperCAmelCase_		)

    # Print and recurse (if needed).
    if isinstance(UpperCAmelCase_ ,     UpperCAmelCase_		):
        if msg is not None:
            print(UpperCAmelCase_		)
        for k in val.keys():
            recursive_print(UpperCAmelCase_ ,     val[k] ,     spaces + 2		)
    elif isinstance(UpperCAmelCase_ ,     torch.Tensor		):
        print(UpperCAmelCase_ ,     """:""" ,     val.size()		)
    else:
        print(UpperCAmelCase_ ,     """:""" ,     UpperCAmelCase_		)
def 							__lowerCAmelCase			(UpperCAmelCase_ ,     UpperCAmelCase_ ,     UpperCAmelCase_ ,     UpperCAmelCase_		):
    # Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
    # for compatibility with later versions of NVIDIA Megatron-LM.
    # The inverse operation is performed inside Megatron-LM to read checkpoints:
    # https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
    # If param is the weight tensor of the self-attention block, the returned tensor
    # will have to be transposed one more time to be read by HuggingFace GPT2.
    a__  			=							param.size()
    # other versions store [num_heads * num_splits * hidden_size, :]
    a__  			=							(num_heads, num_splits, hidden_size) + input_shape[1:]
    a__  			=							param.view(*UpperCAmelCase_		)
    a__  			=							param.transpose(0 ,     1		).contiguous()
    a__  			=							param.view(*UpperCAmelCase_		)
    return param
def 							__lowerCAmelCase			(UpperCAmelCase_ ,     UpperCAmelCase_		):
    a__  			=							config.n_positions
    a__  			=							config.n_layer
    a__  			=							config.vocab_size
    a__  			=							config.n_head
    a__  			=							config.n_embd // config.n_head

    a__  			=							sd_megatron["""model.language_model.embedding.word_embeddings.weight"""][:vocab_size, :]
    a__  			=							{
        """transformer.wte.weight""": word_embeddings,
        """transformer.wpe.weight""": sd_megatron["""model.language_model.embedding.position_embeddings.weight"""],
        """transformer.ln_f.weight""": sd_megatron["""model.language_model.encoder.final_layernorm.weight"""],
        """transformer.ln_f.bias""": sd_megatron["""model.language_model.encoder.final_layernorm.bias"""],
    }

    a__  			=							"""model.language_model.encoder.layers."""
    for i in range(UpperCAmelCase_		):
        a__  			=							torch.tril(torch.ones((n_positions, n_positions) ,     dtype=torch.bool		)		)
        a__  			=							causal_mask.view(1 ,     1 ,     UpperCAmelCase_ ,     UpperCAmelCase_		)
        a__  			=							causal_mask
        a__  			=							torch.tensor(-1E4 ,     dtype=torch.bfloataa		)

        a__  			=							sd_megatron[F"""{pf}{i}.input_layernorm.weight"""]
        a__  			=							sd_megatron[F"""{pf}{i}.input_layernorm.bias"""]

        a__  			=							sd_megatron[F"""{pf}{i}.self_attention.query_key_value.weight"""]
        a__  			=							fix_query_key_value_ordering(UpperCAmelCase_ ,     3 ,     UpperCAmelCase_ ,     UpperCAmelCase_		)
        a__  			=							vala.transpose(0 ,     1		).contiguous()

        a__  			=							sd_megatron[F"""{pf}{i}.self_attention.query_key_value.bias"""]
        a__  			=							fix_query_key_value_ordering(UpperCAmelCase_ ,     3 ,     UpperCAmelCase_ ,     UpperCAmelCase_		)
        a__  			=							vala

        a__  			=							sd_megatron[F"""{pf}{i}.self_attention.dense.weight"""].transpose(
            0 ,     1		)
        a__  			=							sd_megatron[F"""{pf}{i}.self_attention.dense.bias"""]
        a__  			=							sd_megatron[F"""{pf}{i}.post_attention_layernorm.weight"""]
        a__  			=							sd_megatron[F"""{pf}{i}.post_attention_layernorm.bias"""]
        a__  			=							sd_megatron[F"""{pf}{i}.mlp.dense_h_to_4h.weight"""].transpose(0 ,     1		)
        a__  			=							sd_megatron[F"""{pf}{i}.mlp.dense_h_to_4h.bias"""]
        a__  			=							sd_megatron[F"""{pf}{i}.mlp.dense_4h_to_h.weight"""].transpose(
            0 ,     1		)
        a__  			=							sd_megatron[F"""{pf}{i}.mlp.dense_4h_to_h.bias"""]

    # For LM head, transformers' wants the matrix to weight embeddings.
    a__  			=							word_embeddings

    return sd_hf
def 							__lowerCAmelCase			(UpperCAmelCase_ ,     UpperCAmelCase_		):
    a__  			=							64_000
    a__  			=							config_megatron["""encoder_seq_length"""]
    a__  			=							config_megatron["""hidden_size"""]
    a__  			=							config_megatron["""num_layers"""]
    a__  			=							config_megatron["""num_attention_heads"""]
    a__  			=							config_megatron["""ffn_hidden_size"""]
    a__  			=							"""gelu"""
    a__  			=							0.1
    a__  			=							0.1
    a__  			=							0.1
    a__  			=							config_megatron["""layernorm_epsilon"""]  # 1e-5
    a__  			=							config_megatron["""init_method_std"""]  # 0.02
    a__  			=							config_megatron["""apply_query_key_layer_scaling"""]  # True
    a__  			=							True
    a__  			=							True

    # This identifies the 6.7B (7B) model which uses a different tokenizer
    if config_megatron["hidden_size"] == 4_096:
        a__  			=							1  # <|endoftext|>
        a__  			=							1  # <|endoftext|>
        a__  			=							0  # <unk>
    else:
        a__  			=							2  # <s>
        a__  			=							3  # <|endoftext|>
        a__  			=							0  # <pad>

    return config_hf
def 							__lowerCAmelCase			(UpperCAmelCase_		):
    print(UpperCAmelCase_		)

    a__  			=							args.checkpoint_path
    a__  			=							args.save_path
    if isfile(UpperCAmelCase_		):
        raise FileNotFoundError(F"""ERROR! could not find file {checkpoint_path}"""		)

    # Load the model.
    a__  			=							torch.load(UpperCAmelCase_ ,     map_location="""cpu"""		)

    # Load the config.
    a__  			=							checkpoint["""hyper_parameters"""]["""cfg"""]
    a__  			=							GPTaConfig()
    a__  			=							copy_config(config_hf=UpperCAmelCase_ ,     config_megatron=UpperCAmelCase_		)
    a__  			=							["""GPT2LMHeadModel"""]

    a__  			=							checkpoint["""state_dict"""]

    # Convert.
    print("""Converting"""		)
    a__  			=							convert_megatron_checkpoint(UpperCAmelCase_ ,     UpperCAmelCase_		)

    # Print the structure of converted state dict.
    if args.print_checkpoint_structure:
        recursive_print(UpperCAmelCase_ ,     UpperCAmelCase_		)

    a__  			=							"""GPTSw3Tokenizer"""

    # Store the config to file.
    print("""Saving config"""		)
    config_hf.save_pretrained(UpperCAmelCase_		)

    # Store the state_dict to file.
    a__  			=							os.path.join(UpperCAmelCase_ ,     """pytorch_model.bin"""		)
    print(F"""Saving checkpoint to \"{output_checkpoint_file}\""""		)
    torch.save(UpperCAmelCase_ ,     UpperCAmelCase_		)


if __name__ == "__main__":
 SCREAMING_SNAKE_CASE_    :       Dict       =							argparse.ArgumentParser()
 parser.add_argument(
     '''--checkpoint_path''',
     type=str,
     required=True,
     help='''e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000''',
 )
 parser.add_argument('''--save_path''', type=str, required=True, help='''e.g. /home/user/gpt-sw3/hf''')
 parser.add_argument('''--print-checkpoint-structure''', action='''store_true''')
 SCREAMING_SNAKE_CASE_    :       List[str]       =							parser.parse_args()
 main(_args)












import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


SCREAMING_SNAKE_CASE_    :       Tuple       =							logging.get_logger(__name__)







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def __init__(   self			:							int     )      ->      Union[str, Any]:






         """simple docstring"""
         super().__init__()
         if version.parse(torch.__version__     ) < version.parse("""1.12.0"""     ):
             raise ImportError(
                 f"""You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use """
                 """PytorchGELUTanh. Please upgrade torch."""     )






     def 					__SCREAMING_SNAKE_CASE		(   self			:							str    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return nn.functional.gelu(A    ,					approximate="""tanh"""     )







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def 					__SCREAMING_SNAKE_CASE		(   self			:							str    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi     ) * (input + 0.0_4_4_7_1_5 * torch.pow(A    ,					3.0     ))     ))







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def __init__(   self			:							Optional[Any]    ,					A			:							bool = False     )      ->      Optional[Any]:






         """simple docstring"""
         super().__init__()
         if use_gelu_python:
             a__  			=							self._gelu_python
         else:
             a__  			=							nn.functional.gelu

     def 					__SCREAMING_SNAKE_CASE		(   self			:							Dict    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0     )     ))






     def 					__SCREAMING_SNAKE_CASE		(   self			:							Union[str, Any]    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return self.act(A     )







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def 					__SCREAMING_SNAKE_CASE		(   self			:							Any    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return 0.5 * input * (1.0 + torch.tanh(input * 0.7_9_7_8_8_4_5_6_0_8 * (1.0 + 0.0_4_4_7_1_5 * input * input)     ))







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def 					__SCREAMING_SNAKE_CASE		(   self			:							List[Any]    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return input * torch.sigmoid(1.7_0_2 * input     )







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def __init__(   self			:							Tuple    ,					A			:							float    ,					A			:							float     )      ->      Optional[Any]:






         """simple docstring"""
         if min > max:
             raise ValueError(f"""min should be < max (got min: {min}, max: {max})"""     )

         super().__init__()
         a__  			=							min
         a__  			=							max






     def 					__SCREAMING_SNAKE_CASE		(   self			:							Optional[int]    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return torch.clip(gelu(A     )    ,					self.min    ,					self.max     )







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def __init__(   self			:							Tuple     )      ->      List[Any]:






         """simple docstring"""
         super().__init__()
         a__  			=							math.sqrt(2 / math.pi     )






     def 					__SCREAMING_SNAKE_CASE		(   self			:							int    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.0_4_4_7_1_5 * torch.pow(A    ,					3     ))     ))







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def 					__SCREAMING_SNAKE_CASE		(   self			:							Optional[int]    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return nn.functional.silu(A     )







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def __init__(   self			:							Optional[Any]     )      ->      List[str]:






         """simple docstring"""
         super().__init__()
         if version.parse(torch.__version__     ) < version.parse("""1.9.0"""     ):
             a__  			=							self._mish_python
         else:
             a__  			=							nn.functional.mish

     def 					__SCREAMING_SNAKE_CASE		(   self			:							Any    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return input * torch.tanh(nn.functional.softplus(A     )     )






     def 					__SCREAMING_SNAKE_CASE		(   self			:							Union[str, Any]    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return self.act(A     )







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def 					__SCREAMING_SNAKE_CASE		(   self			:							Optional[Any]    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return input







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def 					__SCREAMING_SNAKE_CASE		(   self			:							List[Any]    ,					A			:							str    ,					A			:							Dict=0.7_0_7_1_0_7    ,					A			:							Union[str, Any]=0.2_8_2_0_9_5     )      ->      int:






         """simple docstring"""
         a__  			=							(input - mu).div(sigma * math.sqrt(2.0     )     )
         return 0.5 * (1.0 + torch.erf(A     ))







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def 					__SCREAMING_SNAKE_CASE		(   self			:							List[str]    ,					A			:							List[Any]     )      ->      Tuple:






         """simple docstring"""
         a__  			=							nn.functional.relu(A     )
         a__  			=							torch.square(A     )
         return squared







class 		__UpperCAmelCase(		lowercase__			):





     """simple docstring"""



     def __getitem__(   self			:							List[str]    ,					A			:							int     )      ->      Any:






         """simple docstring"""
         a__  			=							super().__getitem__(A     )
         a__			,		a__  			=							content if isinstance(A    ,					A     ) else (content, {})
         return cls(**A     )






SCREAMING_SNAKE_CASE_    :       List[Any]       =							{
    '''gelu''': GELUActivation,
    '''gelu_10''': (ClippedGELUActivation, {'''min''': -10, '''max''': 10}),
    '''gelu_fast''': FastGELUActivation,
    '''gelu_new''': NewGELUActivation,
    '''gelu_python''': (GELUActivation, {'''use_gelu_python''': True}),
    '''gelu_pytorch_tanh''': PytorchGELUTanh,
    '''gelu_accurate''': AccurateGELUActivation,
    '''laplace''': LaplaceActivation,
    '''linear''': LinearActivation,
    '''mish''': MishActivation,
    '''quick_gelu''': QuickGELUActivation,
    '''relu''': nn.ReLU,
    '''relu2''': ReLUSquaredActivation,
    '''relu6''': nn.ReLUa,
    '''sigmoid''': nn.Sigmoid,
    '''silu''': SiLUActivation,
    '''swish''': SiLUActivation,
    '''tanh''': nn.Tanh,
}
SCREAMING_SNAKE_CASE_    :       Any       =							ClassInstantier(ACTaCLS)
def 							__lowerCAmelCase			(UpperCAmelCase_		):
    if activation_string in ACTaFN:
        return ACTaFN[activation_string]
    else:
        raise KeyError(F"""function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()		)}"""		)


# For backwards compatibility with: from activations import gelu_python
SCREAMING_SNAKE_CASE_    :       Optional[int]       =							get_activation('''gelu_python''')
SCREAMING_SNAKE_CASE_    :       Union[str, Any]       =							get_activation('''gelu_new''')
SCREAMING_SNAKE_CASE_    :       str       =							get_activation('''gelu''')
SCREAMING_SNAKE_CASE_    :       Optional[Any]       =							get_activation('''gelu_fast''')
SCREAMING_SNAKE_CASE_    :       Union[str, Any]       =							get_activation('''quick_gelu''')
SCREAMING_SNAKE_CASE_    :       Tuple       =							get_activation('''silu''')
SCREAMING_SNAKE_CASE_    :       Union[str, Any]       =							get_activation('''mish''')







SCREAMING_SNAKE_CASE_    :       Tuple       =							get_activation('''linear''')






