
import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig




def 						A_			(				snake_case				: Union[str, Any]			,   snake_case				: Optional[Any]			,   snake_case				: Any=0			)   ->     List[str]:



						'''simple docstring'''

						if name is None:
												__UpperCamelCase  	=      None
						else:
												__UpperCamelCase  	=      '''.''' * max(0			,   spaces - 2			) + '''# {:''' + str(50 - spaces			) + '''s}'''
												__UpperCamelCase  	=      fmt.format(snake_case			)

						# Print and recurse (if needed).
						if isinstance(snake_case			,   snake_case			):
												if msg is not None:
																		print(snake_case			)
												for k in val.keys():
																		recursive_print(snake_case			,   val[k]			,   spaces + 2			)
						elif isinstance(snake_case			,   torch.Tensor			):
												print(snake_case			,   ''':'''			,   val.size()			)
						else:
												print(snake_case			,   ''':'''			,   snake_case			)




def 						A_			(				snake_case				: Optional[int]			,   snake_case				: List[Any]			,   snake_case				: Dict			,   snake_case				: Dict			)   ->     List[str]:



						'''simple docstring'''

						__UpperCamelCase  	=      param.size()
						# other versions store [num_heads * num_splits * hidden_size, :]
						__UpperCamelCase  	=      (num_heads, num_splits, hidden_size) + input_shape[1:]
						__UpperCamelCase  	=      param.view(*snake_case			)
						__UpperCamelCase  	=      param.transpose(0			,   1			).contiguous()
						__UpperCamelCase  	=      param.view(*snake_case			)
						return param




def 						A_			(				snake_case				: str			,   snake_case				: str			)   ->     Optional[Any]:



						'''simple docstring'''

						__UpperCamelCase  	=      config.n_positions
						__UpperCamelCase  	=      config.n_layer
						__UpperCamelCase  	=      config.vocab_size
						__UpperCamelCase  	=      config.n_head
						__UpperCamelCase  	=      config.n_embd // config.n_head

						__UpperCamelCase  	=      sd_megatron['''model.language_model.embedding.word_embeddings.weight'''][:vocab_size, :]
						__UpperCamelCase  	=      {
						    '''transformer.wte.weight''': word_embeddings,
						    '''transformer.wpe.weight''': sd_megatron['''model.language_model.embedding.position_embeddings.weight'''],
						    '''transformer.ln_f.weight''': sd_megatron['''model.language_model.encoder.final_layernorm.weight'''],
						    '''transformer.ln_f.bias''': sd_megatron['''model.language_model.encoder.final_layernorm.bias'''],
						}

						__UpperCamelCase  	=      '''model.language_model.encoder.layers.'''
						for i in range(snake_case			):
												__UpperCamelCase  	=      torch.tril(torch.ones((n_positions, n_positions)			,   dtype=torch.bool			)			)
												__UpperCamelCase  	=      causal_mask.view(1			,   1			,   snake_case			,   snake_case			)
												__UpperCamelCase  	=      causal_mask
												__UpperCamelCase  	=      torch.tensor(-1e4			,   dtype=torch.bfloataa			)

												__UpperCamelCase  	=      sd_megatron[f"{pf}{i}.input_layernorm.weight"]
												__UpperCamelCase  	=      sd_megatron[f"{pf}{i}.input_layernorm.bias"]

												__UpperCamelCase  	=      sd_megatron[f"{pf}{i}.self_attention.query_key_value.weight"]
												__UpperCamelCase  	=      fix_query_key_value_ordering(snake_case			,   3			,   snake_case			,   snake_case			)
												__UpperCamelCase  	=      vala.transpose(0			,   1			).contiguous()

												__UpperCamelCase  	=      sd_megatron[f"{pf}{i}.self_attention.query_key_value.bias"]
												__UpperCamelCase  	=      fix_query_key_value_ordering(snake_case			,   3			,   snake_case			,   snake_case			)
												__UpperCamelCase  	=      vala

												__UpperCamelCase  	=      sd_megatron[f"{pf}{i}.self_attention.dense.weight"].transpose(
												    0			,   1			)
												__UpperCamelCase  	=      sd_megatron[f"{pf}{i}.self_attention.dense.bias"]
												__UpperCamelCase  	=      sd_megatron[f"{pf}{i}.post_attention_layernorm.weight"]
												__UpperCamelCase  	=      sd_megatron[f"{pf}{i}.post_attention_layernorm.bias"]
												__UpperCamelCase  	=      sd_megatron[f"{pf}{i}.mlp.dense_h_to_4h.weight"].transpose(0			,   1			)
												__UpperCamelCase  	=      sd_megatron[f"{pf}{i}.mlp.dense_h_to_4h.bias"]
												__UpperCamelCase  	=      sd_megatron[f"{pf}{i}.mlp.dense_4h_to_h.weight"].transpose(
												    0			,   1			)
												__UpperCamelCase  	=      sd_megatron[f"{pf}{i}.mlp.dense_4h_to_h.bias"]

						# For LM head, transformers' wants the matrix to weight embeddings.
						__UpperCamelCase  	=      word_embeddings

						return sd_hf




def 						A_			(				snake_case				: Tuple			,   snake_case				: str			)   ->     Union[str, Any]:



						'''simple docstring'''

						__UpperCamelCase  	=      64000
						__UpperCamelCase  	=      config_megatron['''encoder_seq_length''']
						__UpperCamelCase  	=      config_megatron['''hidden_size''']
						__UpperCamelCase  	=      config_megatron['''num_layers''']
						__UpperCamelCase  	=      config_megatron['''num_attention_heads''']
						__UpperCamelCase  	=      config_megatron['''ffn_hidden_size''']
						__UpperCamelCase  	=      '''gelu'''
						__UpperCamelCase  	=      0.1
						__UpperCamelCase  	=      0.1
						__UpperCamelCase  	=      0.1
						__UpperCamelCase  	=      config_megatron['''layernorm_epsilon''']  # 1e-5
						__UpperCamelCase  	=      config_megatron['''init_method_std''']  # 0.02
						__UpperCamelCase  	=      config_megatron['''apply_query_key_layer_scaling''']  # True
						__UpperCamelCase  	=      True
						__UpperCamelCase  	=      True

						# This identifies the 6.7B (7B) model which uses a different tokenizer
						if config_megatron["hidden_size"] == 4096:
												__UpperCamelCase  	=      1  # <|endoftext|>
												__UpperCamelCase  	=      1  # <|endoftext|>
												__UpperCamelCase  	=      0  # <unk>
						else:
												__UpperCamelCase  	=      2  # <s>
												__UpperCamelCase  	=      3  # <|endoftext|>
												__UpperCamelCase  	=      0  # <pad>

						return config_hf




def 						A_			(				snake_case				: str			)   ->     Any:



						'''simple docstring'''

						print(snake_case			)

						__UpperCamelCase  	=      args.checkpoint_path
						__UpperCamelCase  	=      args.save_path
						if isfile(snake_case			):
												raise FileNotFoundError(f"ERROR! could not find file {checkpoint_path}"			)

						# Load the model.
						__UpperCamelCase  	=      torch.load(snake_case			,   map_location='''cpu'''			)

						# Load the config.
						__UpperCamelCase  	=      checkpoint['''hyper_parameters''']['''cfg''']
						__UpperCamelCase  	=      GPTaConfig()
						__UpperCamelCase  	=      copy_config(config_hf=snake_case			,   config_megatron=snake_case			)
						__UpperCamelCase  	=      ['''GPT2LMHeadModel''']

						__UpperCamelCase  	=      checkpoint['''state_dict''']

						# Convert.
						print('''Converting'''			)
						__UpperCamelCase  	=      convert_megatron_checkpoint(snake_case			,   snake_case			)

						# Print the structure of converted state dict.
						if args.print_checkpoint_structure:
												recursive_print(snake_case			,   snake_case			)

						__UpperCamelCase  	=      '''GPTSw3Tokenizer'''

						# Store the config to file.
						print('''Saving config'''			)
						config_hf.save_pretrained(snake_case			)

						# Store the state_dict to file.
						__UpperCamelCase  	=      os.path.join(snake_case			,   '''pytorch_model.bin'''			)
						print(f"Saving checkpoint to \"{output_checkpoint_file}\""			)
						torch.save(snake_case			,   snake_case			)


if __name__ == "__main__":
				lowercase__				:				Optional[Any]						=      argparse.ArgumentParser()
				parser.add_argument(
				    "--checkpoint_path",
				    type=str,
				    required=True,
				    help="e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000",
				)
				parser.add_argument("--save_path", type=str, required=True, help="e.g. /home/user/gpt-sw3/hf")
				parser.add_argument("--print-checkpoint-structure", action="store_true")
				lowercase__				:				List[Any]						=      parser.parse_args()
				main(_args)





from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


__lowerCAmelCase    :  Optional[int]										= logging.get_logger(__name__)

__lowerCAmelCase    :  Dict										= {
    'google/bigbird-pegasus-large-arxiv': (
        'https://huggingface.co/google/bigbird-pegasus-large-arxiv/resolve/main/config.json'
    ),
    'google/bigbird-pegasus-large-pubmed': (
        'https://huggingface.co/google/bigbird-pegasus-large-pubmed/resolve/main/config.json'
    ),
    'google/bigbird-pegasus-large-bigpatent': (
        'https://huggingface.co/google/bigbird-pegasus-large-bigpatent/resolve/main/config.json'
    ),
    # See all BigBirdPegasus models at https://huggingface.co/models?filter=bigbird_pegasus
}
class 				lowerCamelCase			(							__snake_case					):
      __lowerCamelCase							=					'bigbird_pegasus'
      __lowerCamelCase							=					['past_key_values']
      __lowerCamelCase							=					{
          'num_attention_heads': 'encoder_attention_heads',
          'hidden_size': 'd_model',
          'attention_probs_dropout_prob': 'attention_dropout',
      }






      def __init__(      self  ,		__lowerCamelCase=9_61_03  ,		__lowerCamelCase=40_96  ,		__lowerCamelCase=16  ,		__lowerCamelCase=40_96  ,		__lowerCamelCase=16  ,		__lowerCamelCase=16  ,		__lowerCamelCase=40_96  ,		__lowerCamelCase=16  ,		__lowerCamelCase=0.0  ,		__lowerCamelCase=0.0  ,		__lowerCamelCase=True  ,		__lowerCamelCase=True  ,		__lowerCamelCase="gelu_new"  ,		__lowerCamelCase=10_24  ,		__lowerCamelCase=0.1  ,		__lowerCamelCase=0.0  ,		__lowerCamelCase=0.0  ,		__lowerCamelCase=0.02  ,		__lowerCamelCase=2  ,		__lowerCamelCase=0.0  ,		__lowerCamelCase=True  ,		__lowerCamelCase=0  ,		__lowerCamelCase=2  ,		__lowerCamelCase=1  ,		__lowerCamelCase="block_sparse"  ,		__lowerCamelCase=64  ,		__lowerCamelCase=3  ,		__lowerCamelCase=False  ,		**__lowerCamelCase  ,		)     ->				Dict:


          '''simple docstring'''







          snake_case:						Dict         =   vocab_size
          snake_case:						Union[str, Any]         =   max_position_embeddings
          snake_case:						Tuple         =   d_model
          snake_case:						int         =   encoder_ffn_dim
          snake_case:						List[Any]         =   encoder_layers
          snake_case:						Tuple         =   encoder_attention_heads
          snake_case:						int         =   decoder_ffn_dim
          snake_case:						str         =   decoder_layers
          snake_case:						str         =   decoder_attention_heads
          snake_case:						Optional[Any]         =   dropout
          snake_case:						Union[str, Any]         =   attention_dropout
          snake_case:						List[str]         =   activation_dropout
          snake_case:						Optional[Any]         =   activation_function
          snake_case:						Dict         =   init_std
          snake_case:						str         =   encoder_layerdrop
          snake_case:						List[Any]         =   decoder_layerdrop
          snake_case:						int         =   classifier_dropout
          snake_case:						Optional[Any]         =   use_cache
          snake_case:						int         =   encoder_layers
          snake_case:						str         =   scale_embedding  # scale factor will be sqrt(d_model) if True

          # extra config
          snake_case:						Union[str, Any]         =   attention_type
          snake_case:						Optional[int]         =   block_size
          snake_case:						Optional[int]         =   num_random_blocks
          snake_case:						List[Any]         =   use_bias

          super().__init__(
              pad_token_id=__lowerCamelCase  ,		bos_token_id=__lowerCamelCase  ,		eos_token_id=__lowerCamelCase  ,		is_encoder_decoder=__lowerCamelCase  ,		decoder_start_token_id=__lowerCamelCase  ,		**__lowerCamelCase  ,		)





class 				lowerCamelCase			(							__snake_case					):






      @property
      def 			lowerCAmelCase_   (      self						)     ->				Mapping[str, Mapping[int, str]]:


          '''simple docstring'''







          if self.task in ["default", "seq2seq-lm"]:
              snake_case:						Tuple         =   OrderedDict(
                  [
                      ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
                      ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
                  ]						)

              if self.use_past:
                  snake_case:						List[Any]         =   {0: """batch"""}
                  snake_case:						Optional[int]         =   {0: """batch""", 1: """past_decoder_sequence + sequence"""}
              else:
                  snake_case:						List[Any]         =   {0: """batch""", 1: """decoder_sequence"""}
                  snake_case:						Union[str, Any]         =   {0: """batch""", 1: """decoder_sequence"""}

              if self.use_past:
                  self.fill_with_past_key_values_(__lowerCamelCase  ,		direction="""inputs"""						)
          elif self.task == "causal-lm":
              # TODO: figure this case out.
              snake_case:						List[str]         =   OrderedDict(
                  [
                      ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
                      ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
                  ]						)
              if self.use_past:
                  snake_case    , snake_case:						str         =   self.num_layers
                  for i in range(__lowerCamelCase						):
                      snake_case:						Any         =   {0: """batch""", 2: """past_sequence + sequence"""}
                      snake_case:						str         =   {0: """batch""", 2: """past_sequence + sequence"""}
          else:
              snake_case:						str         =   OrderedDict(
                  [
                      ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
                      ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
                      ("""decoder_input_ids""", {0: """batch""", 1: """decoder_sequence"""}),
                      ("""decoder_attention_mask""", {0: """batch""", 1: """decoder_sequence"""}),
                  ]						)

          return common_inputs






      @property
      def 			lowerCAmelCase_   (      self						)     ->				Mapping[str, Mapping[int, str]]:


          '''simple docstring'''







          if self.task in ["default", "seq2seq-lm"]:
              snake_case:						List[str]         =   super().outputs
          else:
              snake_case:						Optional[Any]         =   super(__lowerCamelCase  ,		self						).outputs
              if self.use_past:
                  snake_case    , snake_case:						str         =   self.num_layers
                  for i in range(__lowerCamelCase						):
                      snake_case:						List[Any]         =   {0: """batch""", 2: """past_sequence + sequence"""}
                      snake_case:						int         =   {0: """batch""", 2: """past_sequence + sequence"""}
          return common_outputs






      def 			lowerCAmelCase_   (      self  ,		__lowerCamelCase  ,		__lowerCamelCase = -1  ,		__lowerCamelCase = -1  ,		__lowerCamelCase = False  ,		__lowerCamelCase = None  ,		)     ->				Mapping[str, Any]:


          '''simple docstring'''







          snake_case:						List[Any]         =   self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
              __lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase						)

          # Generate decoder inputs
          snake_case:						Any         =   seq_length if not self.use_past else 1
          snake_case:						List[str]         =   self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
              __lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase						)
          snake_case:						Dict         =   {F"decoder_{name}": tensor for name, tensor in decoder_inputs.items()}
          snake_case:						List[str]         =   dict(**__lowerCamelCase  ,		**__lowerCamelCase						)

          if self.use_past:
              if not is_torch_available():
                  raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""						)
              else:
                  import torch
              snake_case    , snake_case:						Optional[int]         =   common_inputs["""input_ids"""].shape
              snake_case:						Optional[Any]         =   common_inputs["""decoder_input_ids"""].shape[1]
              snake_case    , snake_case:						Any         =   self.num_attention_heads
              snake_case:						Optional[int]         =   (
                  batch,
                  num_encoder_attention_heads,
                  encoder_seq_length,
                  self._config.hidden_size // num_encoder_attention_heads,
              )
              snake_case:						List[Any]         =   decoder_seq_length + 3
              snake_case:						Optional[Any]         =   (
                  batch,
                  num_decoder_attention_heads,
                  decoder_past_length,
                  self._config.hidden_size // num_decoder_attention_heads,
              )

              snake_case:						Tuple         =   torch.cat(
                  [common_inputs["""decoder_attention_mask"""], torch.ones(__lowerCamelCase  ,		__lowerCamelCase						)]  ,		dim=1						)

              snake_case:						Tuple         =   []
              # If the number of encoder and decoder layers are present in the model configuration, both are considered
              snake_case    , snake_case:						Union[str, Any]         =   self.num_layers
              snake_case:						List[str]         =   min(__lowerCamelCase  ,		__lowerCamelCase						)
              snake_case:						List[str]         =   max(__lowerCamelCase  ,		__lowerCamelCase						) - min_num_layers
              snake_case:						Union[str, Any]         =   """encoder""" if num_encoder_layers > num_decoder_layers else """decoder"""

              for _ in range(__lowerCamelCase						):
                  common_inputs["past_key_values"].append(
                      (
                          torch.zeros(__lowerCamelCase						),
                          torch.zeros(__lowerCamelCase						),
                          torch.zeros(__lowerCamelCase						),
                          torch.zeros(__lowerCamelCase						),
                      )						)
              # TODO: test this.
              snake_case:						Dict         =   encoder_shape if remaining_side_name == """encoder""" else decoder_shape
              for _ in range(__lowerCamelCase  ,		__lowerCamelCase						):
                  common_inputs["past_key_values"].append((torch.zeros(__lowerCamelCase						), torch.zeros(__lowerCamelCase						))						)
          return common_inputs






      def 			lowerCAmelCase_   (      self  ,		__lowerCamelCase  ,		__lowerCamelCase = -1  ,		__lowerCamelCase = -1  ,		__lowerCamelCase = False  ,		__lowerCamelCase = None  ,		)     ->				Mapping[str, Any]:


          '''simple docstring'''







          snake_case:						Any         =   self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
              __lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase						)

          if self.use_past:
              if not is_torch_available():
                  raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""						)
              else:
                  import torch
              snake_case    , snake_case:						Dict         =   common_inputs["""input_ids"""].shape
              # Not using the same length for past_key_values
              snake_case:						int         =   seqlen + 2
              snake_case    , snake_case:						Dict         =   self.num_layers
              snake_case    , snake_case:						Tuple         =   self.num_attention_heads
              snake_case:						Optional[int]         =   (
                  batch,
                  num_encoder_attention_heads,
                  past_key_values_length,
                  self._config.hidden_size // num_encoder_attention_heads,
              )

              snake_case:						int         =   common_inputs["""attention_mask"""].dtype
              snake_case:						Union[str, Any]         =   torch.cat(
                  [common_inputs["""attention_mask"""], torch.ones(__lowerCamelCase  ,		__lowerCamelCase  ,		dtype=__lowerCamelCase						)]  ,		dim=1						)
              snake_case:						Dict         =   [
                  (torch.zeros(__lowerCamelCase						), torch.zeros(__lowerCamelCase						)) for _ in range(__lowerCamelCase						)
              ]
          return common_inputs






      def 			lowerCAmelCase_   (      self  ,		__lowerCamelCase  ,		__lowerCamelCase = -1  ,		__lowerCamelCase = -1  ,		__lowerCamelCase = False  ,		__lowerCamelCase = None  ,		)     ->				Mapping[str, Any]:


          '''simple docstring'''







          snake_case:						str         =   compute_effective_axis_dimension(
              __lowerCamelCase  ,		fixed_dimension=OnnxConfig.default_fixed_batch  ,		num_token_to_add=0						)

          # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
          snake_case:						str         =   tokenizer.num_special_tokens_to_add(__lowerCamelCase						)
          snake_case:						int         =   compute_effective_axis_dimension(
              __lowerCamelCase  ,		fixed_dimension=OnnxConfig.default_fixed_sequence  ,		num_token_to_add=__lowerCamelCase						)

          # Generate dummy inputs according to compute batch and sequence
          snake_case:						Dict         =   [""" """.join([tokenizer.unk_token]						) * seq_length] * batch_size
          snake_case:						Any         =   dict(tokenizer(__lowerCamelCase  ,		return_tensors=__lowerCamelCase						)						)
          return common_inputs






      def 			lowerCAmelCase_   (      self  ,		__lowerCamelCase  ,		__lowerCamelCase = -1  ,		__lowerCamelCase = -1  ,		__lowerCamelCase = False  ,		__lowerCamelCase = None  ,		)     ->				Mapping[str, Any]:


          '''simple docstring'''







          if self.task in ["default", "seq2seq-lm"]:
              snake_case:						Union[str, Any]         =   self._generate_dummy_inputs_for_default_and_seqaseq_lm(
                  __lowerCamelCase  ,		batch_size=__lowerCamelCase  ,		seq_length=__lowerCamelCase  ,		is_pair=__lowerCamelCase  ,		framework=__lowerCamelCase						)

          elif self.task == "causal-lm":
              snake_case:						Union[str, Any]         =   self._generate_dummy_inputs_for_causal_lm(
                  __lowerCamelCase  ,		batch_size=__lowerCamelCase  ,		seq_length=__lowerCamelCase  ,		is_pair=__lowerCamelCase  ,		framework=__lowerCamelCase						)
          else:
              snake_case:						int         =   self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                  __lowerCamelCase  ,		batch_size=__lowerCamelCase  ,		seq_length=__lowerCamelCase  ,		is_pair=__lowerCamelCase  ,		framework=__lowerCamelCase						)

          return common_inputs






      def 			lowerCAmelCase_   (      self  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase						)     ->				List[Any]:


          '''simple docstring'''







          if self.task in ["default", "seq2seq-lm"]:
              snake_case:						Optional[int]         =   super()._flatten_past_key_values_(__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase						)
          else:
              snake_case:						List[str]         =   super(__lowerCamelCase  ,		self						)._flatten_past_key_values_(
                  __lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase						)


