






"""simple docstring"""
import argparse
import json

from transformers import GPTNeoConfig, GPTNeoForCausalLM, load_tf_weights_in_gpt_neo
from transformers.utils import logging


logging.set_verbosity_info()






def 			lowerCamelCase_    (		_lowerCamelCase     ,  _lowerCamelCase     ,  _lowerCamelCase							):
 # Initialise PyTorch model
 lowerCamelCase__	:     Tuple								=		json.load(open(_lowerCamelCase     ,  'r'							)							)
 lowerCamelCase__	:     Dict								=		GPTNeoConfig(
     hidden_size=config_json['n_embd']     ,  num_layers=config_json['n_layer']     ,  num_heads=config_json['n_head']     ,  attention_types=config_json['attention_types']     ,  max_position_embeddings=config_json['n_positions']     ,  resid_dropout=config_json['res_dropout']     ,  embed_dropout=config_json['embed_dropout']     ,  attention_dropout=config_json['attn_dropout']     ,  )
 print(f'''Building PyTorch model from configuration: {config}'''							)
 lowerCamelCase__	:     str								=		GPTNeoForCausalLM(_lowerCamelCase							)

 # Load weights from tf checkpoint
 load_tf_weights_in_gpt_neo(_lowerCamelCase     ,  _lowerCamelCase     ,  _lowerCamelCase							)

 # Save pytorch-model
 print(f'''Save PyTorch model to {pytorch_dump_path}'''							)
 model.save_pretrained(_lowerCamelCase							)


if __name__ == "__main__":
      A_     :					int							=     argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          "--tf_checkpoint_path", default=None, type=str, required=True, help="Path to the TensorFlow checkpoint path."
      )
      parser.add_argument(
          "--config_file",
          default=None,
          type=str,
          required=True,
          help=(
              "The config json file corresponding to the pre-trained mesh-tf model. \n"
              "This specifies the model architecture."
          ),
      )
      parser.add_argument(
          "--pytorch_dump_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
      )
      A_     :					Any							=     parser.parse_args()
      convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.config_file, args.pytorch_dump_path)









"""simple docstring"""
import torch.nn as nn

from .imports import is_fpa_available


if is_fpa_available():
      import transformer_engine.pytorch as te






def 			lowerCamelCase_    (		_lowerCamelCase     ,  _lowerCamelCase=True     ,  _lowerCamelCase=True     ,  _lowerCamelCase=True							):
 if not is_fpa_available():
  raise ImportError('Using `convert_model` requires transformer_engine to be installed.'							)
 for name, module in model.named_children():
  if isinstance(_lowerCamelCase     ,  nn.Linear							) and to_transformer_engine and _convert_linear:
   # Return early if the linear layer weights are not multiples of 16
   if any(p % 16 != 0 for p in module.weight.shape							):
    return
   lowerCamelCase__	:     List[Any]								=		module.bias is not None
   lowerCamelCase__	:     Tuple								=		te.Linear(
       module.in_features     ,  module.out_features     ,  bias=_lowerCamelCase     ,  params_dtype=module.weight.dtype							)
   lowerCamelCase__	:     Optional[Any]								=		module.weight.data.clone()
   if has_bias:
    lowerCamelCase__	:     List[str]								=		module.bias.data.clone()

   setattr(_lowerCamelCase     ,  _lowerCamelCase     ,  _lowerCamelCase							)
  elif isinstance(_lowerCamelCase     ,  nn.LayerNorm							) and to_transformer_engine and _convert_ln:
   lowerCamelCase__	:     Tuple								=		te.LayerNorm(module.normalized_shape[0]     ,  eps=module.eps     ,  params_dtype=module.weight.dtype							)
   lowerCamelCase__	:     Tuple								=		module.weight.data.clone()
   lowerCamelCase__	:     Optional[Any]								=		module.bias.data.clone()

   setattr(_lowerCamelCase     ,  _lowerCamelCase     ,  _lowerCamelCase							)
  elif isinstance(_lowerCamelCase     ,  te.Linear							) and not to_transformer_engine and _convert_linear:
   lowerCamelCase__	:     Optional[Any]								=		module.bias is not None
   lowerCamelCase__	:     Union[str, Any]								=		nn.Linear(
       module.in_features     ,  module.out_features     ,  bias=_lowerCamelCase     ,  params_dtype=module.weight.dtype							)
   lowerCamelCase__	:     str								=		module.weight.data.clone()
   if has_bias:
    lowerCamelCase__	:     Optional[int]								=		module.bias.data.clone()

   setattr(_lowerCamelCase     ,  _lowerCamelCase     ,  _lowerCamelCase							)
  elif isinstance(_lowerCamelCase     ,  te.LayerNorm							) and not to_transformer_engine and _convert_ln:
   lowerCamelCase__	:     str								=		nn.LayerNorm(module.normalized_shape[0]     ,  eps=module.eps     ,  params_dtype=module.weight.dtype							)
   lowerCamelCase__	:     Dict								=		module.weight.data.clone()
   lowerCamelCase__	:     Dict								=		module.bias.data.clone()

   setattr(_lowerCamelCase     ,  _lowerCamelCase     ,  _lowerCamelCase							)
  else:
   convert_model(
       _lowerCamelCase     ,  to_transformer_engine=_lowerCamelCase     ,  _convert_linear=_convert_linear     ,  _convert_ln=_convert_ln     ,  )






def 			lowerCamelCase_    (		_lowerCamelCase							):
 if not is_fpa_available():
  raise ImportError('Using `has_transformer_engine_layers` requires transformer_engine to be installed.'							)
 for m in model.modules():
  if isinstance(_lowerCamelCase     ,  (te.LayerNorm, te.Linear)							):
   return True
 return False

