"""simple docstring"""






import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig



def SCREAMING_SNAKE_CASE__			(   __UpperCAmelCase       ,    __UpperCAmelCase       ,    __UpperCAmelCase=0 )  ->  List[Any]:
   # Format the message.
   if name is None:
      lowercase__:   Any					=      None
   else:
      lowercase__:   Optional[Any]					=      '''.''' * max(0       ,    spaces - 2 ) + '''# {:''' + str(5_0 - spaces ) + '''s}'''
      lowercase__:   List[str]					=      fmt.format(__UpperCAmelCase )

   # Print and recurse (if needed).
   if isinstance(__UpperCAmelCase       ,    __UpperCAmelCase ):
      if msg is not None:
         print(__UpperCAmelCase )
      for k in val.keys():
         recursive_print(__UpperCAmelCase       ,    val[k]       ,    spaces + 2 )
   elif isinstance(__UpperCAmelCase       ,    torch.Tensor ):
      print(__UpperCAmelCase       ,    ''':'''       ,    val.size() )
   else:
      print(__UpperCAmelCase       ,    ''':'''       ,    __UpperCAmelCase )



def SCREAMING_SNAKE_CASE__			(   __UpperCAmelCase       ,    __UpperCAmelCase       ,    __UpperCAmelCase       ,    __UpperCAmelCase       ,    __UpperCAmelCase )  ->  Any:
   # Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
   # for compatibility with later versions of NVIDIA Megatron-LM.
   # The inverse operation is performed inside Megatron-LM to read checkpoints:
   # https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
   # If param is the weight tensor of the self-attention block, the returned tensor
   # will have to be transposed one more time to be read by HuggingFace GPT2.
   lowercase__:   str					=      param.size()
   if checkpoint_version == 1.0:
      # version 1.0 stores [num_heads * hidden_size * num_splits, :]
      lowercase__:   List[str]					=      (num_heads, hidden_size, num_splits) + input_shape[1:]
      lowercase__:   Optional[Any]					=      param.view(*__UpperCAmelCase )
      lowercase__:   str					=      param.transpose(0       ,    2 )
      lowercase__:   Union[str, Any]					=      param.transpose(1       ,    2 ).contiguous()
   elif checkpoint_version >= 2.0:
      # other versions store [num_heads * num_splits * hidden_size, :]
      lowercase__:   int					=      (num_heads, num_splits, hidden_size) + input_shape[1:]
      lowercase__:   Optional[int]					=      param.view(*__UpperCAmelCase )
      lowercase__:   Dict					=      param.transpose(0       ,    1 ).contiguous()
   lowercase__:   Union[str, Any]					=      param.view(*__UpperCAmelCase )
   return param



def SCREAMING_SNAKE_CASE__			(   __UpperCAmelCase       ,    __UpperCAmelCase       ,    __UpperCAmelCase )  ->  str:
   # The converted output model.
   lowercase__:   int					=      {}

   # old versions did not store training args
   lowercase__:   int					=      input_state_dict.get('''args'''       ,    __UpperCAmelCase )
   if ds_args is not None:
      # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
      # from pprint import pprint
      # pprint(vars(ds_args))

      lowercase__:   List[str]					=      ds_args.padded_vocab_size
      lowercase__:   int					=      ds_args.max_position_embeddings
      lowercase__:   int					=      ds_args.hidden_size
      lowercase__:   Optional[Any]					=      ds_args.num_layers
      lowercase__:   int					=      ds_args.num_attention_heads
      lowercase__:   Optional[Any]					=      ds_args.ffn_hidden_size
      # pprint(config)

   # The number of heads.
   lowercase__:   List[Any]					=      config.n_head
   # The hidden_size per head.
   lowercase__:   List[Any]					=      config.n_embd // config.n_head
   # Megatron-LM checkpoint version
   if "checkpoint_version" in input_state_dict.keys():
      lowercase__:   int					=      input_state_dict['''checkpoint_version''']
   else:
      lowercase__:   List[Any]					=      0.0

   # The model.
   lowercase__:   Any					=      input_state_dict['''model''']
   # The language model.
   lowercase__:   Dict					=      model['''language_model''']
   # The embeddings.
   lowercase__:   str					=      lm['''embedding''']

   # The word embeddings.
   lowercase__:   Any					=      embeddings['''word_embeddings''']['''weight''']
   # Truncate the embedding table to vocab_size rows.
   lowercase__:   List[Any]					=      word_embeddings[: config.vocab_size, :]
   lowercase__:   List[str]					=      word_embeddings

   # The position embeddings.
   lowercase__:   Dict					=      embeddings['''position_embeddings''']['''weight''']
   # Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
   lowercase__:   int					=      pos_embeddings.size(0 )
   if n_positions != config.n_positions:
      raise ValueError(
          F"""pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match""" )
   # Store the position embeddings.
   lowercase__:   Optional[Any]					=      pos_embeddings

   # The transformer.
   lowercase__:   Any					=      lm['''transformer'''] if '''transformer''' in lm.keys() else lm['''encoder''']

   # The regex to extract layer names.
   lowercase__:   Optional[int]					=      re.compile(R'''layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)''' )

   # The simple map of names for "automated" rules.
   lowercase__:   Tuple					=      {
       '''attention.dense''': '''.attn.c_proj.''',
       '''self_attention.dense''': '''.attn.c_proj.''',
       '''mlp.dense_h_to_4h''': '''.mlp.c_fc.''',
       '''mlp.dense_4h_to_h''': '''.mlp.c_proj.''',
   }

   # Extract the layers.
   for key, val in transformer.items():
      # Match the name.
      lowercase__:   List[Any]					=      layer_re.match(__UpperCAmelCase )

      # Stop if that's not a layer
      if m is None:
         break

      # The index of the layer.
      lowercase__:   List[Any]					=      int(m.group(1 ) )
      # The name of the operation.
      lowercase__:   Optional[int]					=      m.group(2 )
      # Is it a weight or a bias?
      lowercase__:   Optional[int]					=      m.group(3 )

      # The name of the layer.
      lowercase__:   List[Any]					=      F"""transformer.h.{layer_idx}"""

      # For layernorm(s), simply store the layer norm.
      if op_name.endswith('''layernorm''' ):
         lowercase__:   Optional[int]					=      '''ln_1''' if op_name.startswith('''input''' ) else '''ln_2'''
         lowercase__:   int					=      val

      # Transpose the QKV matrix.
      elif (
          op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
      ) and weight_or_bias == "weight":
         # Insert a tensor of 1x1xDxD bias.
         lowercase__:   int					=      torch.tril(torch.ones((n_positions, n_positions)       ,    dtype=torch.floataa ) ).view(
             1       ,    1       ,    __UpperCAmelCase       ,    __UpperCAmelCase )
         lowercase__:   List[str]					=      causal_mask

         # Insert a "dummy" tensor for masked_bias.
         lowercase__:   List[str]					=      torch.tensor(-1e4       ,    dtype=torch.floataa )
         lowercase__:   int					=      masked_bias

         lowercase__:   Optional[int]					=      fix_query_key_value_ordering(__UpperCAmelCase       ,    __UpperCAmelCase       ,    3       ,    __UpperCAmelCase       ,    __UpperCAmelCase )
         # Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
         lowercase__:   str					=      out_val.transpose(0       ,    1 ).contiguous()
         # Store.
         lowercase__:   Union[str, Any]					=      out_val

      # Transpose the bias.
      elif (
          op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
      ) and weight_or_bias == "bias":
         lowercase__:   List[str]					=      fix_query_key_value_ordering(__UpperCAmelCase       ,    __UpperCAmelCase       ,    3       ,    __UpperCAmelCase       ,    __UpperCAmelCase )
         # Store. No change of shape.
         lowercase__:   Tuple					=      out_val

      # Transpose the weights.
      elif weight_or_bias == "weight":
         lowercase__:   Any					=      megatron_to_transformers[op_name]
         lowercase__:   Union[str, Any]					=      val.transpose(0       ,    1 )

      # Copy the bias.
      elif weight_or_bias == "bias":
         lowercase__:   Optional[int]					=      megatron_to_transformers[op_name]
         lowercase__:   Any					=      val

    # DEBUG.
   assert config.n_layer == layer_idx + 1

   # The final layernorm.
   lowercase__:   str					=      transformer['''final_layernorm.weight''']
   lowercase__:   str					=      transformer['''final_layernorm.bias''']

   # For LM head, transformers' wants the matrix to weight embeddings.
   lowercase__:   List[str]					=      word_embeddings

   # It should be done!
   return output_state_dict



def SCREAMING_SNAKE_CASE__			(   )  ->  Dict:
   # Create the argument parser.
   lowercase__:   Dict					=      argparse.ArgumentParser()
   parser.add_argument('''--print-checkpoint-structure'''       ,    action='''store_true''' )
   parser.add_argument(
       '''path_to_checkpoint'''       ,    type=__UpperCAmelCase       ,    help='''Path to the checkpoint file (.zip archive or direct .pt file)'''       ,    )
   parser.add_argument(
       '''--config_file'''       ,    default=''''''       ,    type=__UpperCAmelCase       ,    help='''An optional config json file describing the pre-trained model.'''       ,    )
   lowercase__:   str					=      parser.parse_args()

   # Extract the basename.
   lowercase__:   int					=      os.path.dirname(args.path_to_checkpoint )

   # Load the model.
   # the .zip is very optional, let's keep it for backward compatibility
   print(F"""Extracting PyTorch state dictionary from {args.path_to_checkpoint}""" )
   if args.path_to_checkpoint.endswith('''.zip''' ):
      with zipfile.ZipFile(args.path_to_checkpoint       ,    '''r''' ) as checkpoint:
         with checkpoint.open('''release/mp_rank_00/model_optim_rng.pt''' ) as pytorch_dict:
            lowercase__:   Any					=      torch.load(__UpperCAmelCase       ,    map_location='''cpu''' )
   else:
      lowercase__:   List[str]					=      torch.load(args.path_to_checkpoint       ,    map_location='''cpu''' )

   lowercase__:   Optional[int]					=      input_state_dict.get('''args'''       ,    __UpperCAmelCase )

   # Read the config, or default to the model released by NVIDIA.
   if args.config_file == "":
      if ds_args is not None:
         if ds_args.bias_gelu_fusion:
            lowercase__:   Optional[Any]					=      '''gelu_fast'''
         elif ds_args.openai_gelu:
            lowercase__:   Optional[Any]					=      '''gelu_new'''
         else:
            lowercase__:   Dict					=      '''gelu'''
      else:
         # in the very early days this used to be "gelu_new"
         lowercase__:   Optional[Any]					=      '''gelu_new'''

      # Spell out all parameters in case the defaults change.
      lowercase__:   Dict					=      GPTaConfig(
          vocab_size=5_0_2_5_7       ,    n_positions=1_0_2_4       ,    n_embd=1_0_2_4       ,    n_layer=2_4       ,    n_head=1_6       ,    n_inner=4_0_9_6       ,    activation_function=__UpperCAmelCase       ,    resid_pdrop=0.1       ,    embd_pdrop=0.1       ,    attn_pdrop=0.1       ,    layer_norm_epsilon=1e-5       ,    initializer_range=0.0_2       ,    summary_type='''cls_index'''       ,    summary_use_proj=__UpperCAmelCase       ,    summary_activation=__UpperCAmelCase       ,    summary_proj_to_labels=__UpperCAmelCase       ,    summary_first_dropout=0.1       ,    scale_attn_weights=__UpperCAmelCase       ,    use_cache=__UpperCAmelCase       ,    bos_token_id=5_0_2_5_6       ,    eos_token_id=5_0_2_5_6       ,    )
   else:
      lowercase__:   Any					=      GPTaConfig.from_json_file(args.config_file )

   lowercase__:   List[str]					=      ['''GPT2LMHeadModel''']

   # Convert.
   print('''Converting''' )
   lowercase__:   Tuple					=      convert_megatron_checkpoint(__UpperCAmelCase       ,    __UpperCAmelCase       ,    __UpperCAmelCase )

   # Print the structure of converted state dict.
   if args.print_checkpoint_structure:
      recursive_print(__UpperCAmelCase       ,    __UpperCAmelCase )

   # Add tokenizer class info to config
   # see https://github.com/huggingface/transformers/issues/13906)
   if ds_args is not None:
      lowercase__:   Dict					=      ds_args.tokenizer_type
      if tokenizer_type == "GPT2BPETokenizer":
         lowercase__:   List[Any]					=      '''gpt2'''
      elif tokenizer_type == "PretrainedFromHF":
         lowercase__:   Optional[Any]					=      ds_args.tokenizer_name_or_path
      else:
         raise ValueError(F"""Unrecognized tokenizer_type {tokenizer_type}""" )
   else:
      lowercase__:   Dict					=      '''gpt2'''

   lowercase__:   Optional[int]					=      AutoTokenizer.from_pretrained(__UpperCAmelCase )
   lowercase__:   str					=      type(__UpperCAmelCase ).__name__
   lowercase__:   Dict					=      tokenizer_class

   # Store the config to file.
   print('''Saving config''' )
   config.save_pretrained(__UpperCAmelCase )

   # Save tokenizer based on args
   print(F"""Adding {tokenizer_class} tokenizer files""" )
   tokenizer.save_pretrained(__UpperCAmelCase )

   # Store the state_dict to file.
   lowercase__:   str					=      os.path.join(__UpperCAmelCase       ,    '''pytorch_model.bin''' )
   print(F"""Saving checkpoint to \"{output_checkpoint_file}\"""" )
   torch.save(__UpperCAmelCase       ,    __UpperCAmelCase )


####################################################################################################

if __name__ == "__main__":
       main()

####################################################################################################




'''simple docstring'''




def 	a_					( lowerCamelCase	:     Optional[Any]				):
   return [
       {
           0: [1, 2],
           1: [0, 2],
           2: [0, 1, 3, 5],
           3: [2, 4],
           4: [3],
           5: [2, 6, 8],
           6: [5, 7],
           7: [6, 8],
           8: [5, 7],
       },
       {
           0: [6],
           1: [9],
           2: [4, 5],
           3: [4],
           4: [2, 3],
           5: [2],
           6: [0, 7],
           7: [6],
           8: [],
           9: [1],
       },
       {
           0: [4],
           1: [6],
           2: [],
           3: [5, 6, 7],
           4: [0, 6],
           5: [3, 8, 9],
           6: [1, 3, 4, 7],
           7: [3, 6, 8, 9],
           8: [5, 7],
           9: [5, 7],
       },
       {
           0: [1, 3],
           1: [0, 2, 4],
           2: [1, 3, 4],
           3: [0, 2, 4],
           4: [1, 2, 3],
       },
   ][index]





def 	a_					( lowerCamelCase	:     dict[int, list[int]]				):
   lowerCAmelCase     =			0
   lowerCAmelCase     =			len(lowerCamelCase				)  # No of vertices in graph
   lowerCAmelCase     =			[0] * n
   lowerCAmelCase     =			[False] * n

   def dfs(lowerCamelCase	:     Tuple	, lowerCamelCase	:     str	, lowerCamelCase	:     Dict	, lowerCamelCase	:     str				):
      lowerCAmelCase     =			True
      lowerCAmelCase     =			id_
      id_ += 1
      for to in graph[at]:
         if to == parent:
            pass
         elif not visited[to]:
            dfs(lowerCamelCase	, lowerCamelCase	, lowerCamelCase	, id_				)
            lowerCAmelCase     =			min(low[at]	, low[to]				)
            if id_ <= low[to]:
               bridges.append((at, to) if at < to else (to, at)				)
         else:
            # This edge is a back edge and cannot be a bridge
            lowerCAmelCase     =			min(low[at]	, low[to]				)

   lowerCAmelCase     =			[]
   for i in range(lowerCamelCase				):
      if not visited[i]:
         dfs(lowerCamelCase	, -1	, lowerCamelCase	, id_				)
   return bridges


if __name__ == "__main__":
       import doctest

       doctest.testmod()





