"""simple docstring"""




import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
     from transformers import LlamaTokenizerFast
except ImportError as e:
     warnings.warn(e)
     warnings.warn(
         'The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'
     )
     _a      :		Optional[int]           =   None
_a      :		Optional[int]           =   {
    '7B': 11_008,
    '13B': 13_824,
    '30B': 17_920,
    '65B': 22_016,
    '70B': 28_672,
}
_a      :		int           =   {
    '7B': 1,
    '7Bf': 1,
    '13B': 2,
    '13Bf': 2,
    '30B': 4,
    '65B': 8,
    '70B': 8,
    '70Bf': 8,
}





def 							SCREAMING_SNAKE_CASE     (			_lowerCamelCase						:       Dict			,_lowerCamelCase						:       Dict=1			,_lowerCamelCase						:       Optional[int]=256	)       ->      Optional[Any]:
  return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3	)	) + multiple_of - 1) // multiple_of)





def 							SCREAMING_SNAKE_CASE     (			_lowerCamelCase						:       List[str]	)       ->      Any:
  with open(_lowerCamelCase			,"""r"""	) as f:
    return json.load(_lowerCamelCase	)





def 							SCREAMING_SNAKE_CASE     (			_lowerCamelCase						:       int			,_lowerCamelCase						:       Optional[Any]	)       ->      List[str]:
  with open(_lowerCamelCase			,"""w"""	) as f:
    json.dump(_lowerCamelCase			,_lowerCamelCase	)





def 							SCREAMING_SNAKE_CASE     (			_lowerCamelCase						:       List[Any]			,_lowerCamelCase						:       List[Any]			,_lowerCamelCase						:       Optional[Any]			,_lowerCamelCase						:       Dict=True	)       ->      Dict:
  os.makedirs(_lowerCamelCase			,exist_ok=_lowerCamelCase	)
  _lowerCAmelCase						:       Optional[Any]  			=     os.path.join(_lowerCamelCase			,"""tmp"""	)
  os.makedirs(_lowerCamelCase			,exist_ok=_lowerCamelCase	)

  _lowerCAmelCase						:       Optional[int]  			=     read_json(os.path.join(_lowerCamelCase			,"""params.json"""	)	)
  _lowerCAmelCase						:       Optional[int]  			=     NUM_SHARDS[model_size]
  _lowerCAmelCase						:       int  			=     params["""n_layers"""]
  _lowerCAmelCase						:       str  			=     params["""n_heads"""]
  _lowerCAmelCase						:       Dict  			=     n_heads // num_shards
  _lowerCAmelCase						:       Tuple  			=     params["""dim"""]
  _lowerCAmelCase						:       Any  			=     dim // n_heads
  _lowerCAmelCase						:       Dict  			=     1_00_00.0
  _lowerCAmelCase						:       Optional[int]  			=     1.0 / (base ** (torch.arange(0			,_lowerCamelCase			,2	).float() / dims_per_head))

  if "n_kv_heads" in params:
    _lowerCAmelCase						:       Any  			=     params["""n_kv_heads"""]  # for GQA / MQA
    _lowerCAmelCase						:       int  			=     n_heads_per_shard // num_key_value_heads
    _lowerCAmelCase						:       Optional[int]  			=     dim // num_key_value_heads
  else:  # compatibility with other checkpoints
    _lowerCAmelCase						:       List[str]  			=     n_heads
    _lowerCAmelCase						:       List[str]  			=     n_heads_per_shard
    _lowerCAmelCase						:       str  			=     dim

  # permute for sliced rotary
  def permute(_lowerCamelCase						:       Dict			,_lowerCamelCase						:       int=n_heads			,_lowerCamelCase						:       List[str]=dim			,_lowerCamelCase						:       List[str]=dim	):
    return w.view(_lowerCamelCase			,dima // n_heads // 2			,2			,_lowerCamelCase	).transpose(1			,2	).reshape(_lowerCamelCase			,_lowerCamelCase	)

  print(f"Fetching all parameters from the checkpoint at {input_base_path}."	)
  # Load weights
  if model_size == "7B":
    # Not sharded
    # (The sharded implementation would also work, but this is simpler.)
    _lowerCAmelCase						:       Any  			=     torch.load(os.path.join(_lowerCamelCase			,"""consolidated.00.pth"""	)			,map_location="""cpu"""	)
  else:
    # Sharded
    _lowerCAmelCase						:       List[Any]  			=     [
        torch.load(os.path.join(_lowerCamelCase			,f"consolidated.{i:02d}.pth"	)			,map_location="""cpu"""	)
        for i in range(_lowerCamelCase	)
    ]
  _lowerCAmelCase						:       Any  			=     0
  _lowerCAmelCase						:       Dict  			=     {"""weight_map""": {}}
  for layer_i in range(_lowerCamelCase	):
    _lowerCAmelCase						:       int  			=     f"pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"
    if model_size == "7B":
      # Unsharded
      _lowerCAmelCase						:       Optional[Any]  			=     {
          f"model.layers.{layer_i}.self_attn.q_proj.weight": permute(
              loaded[f"layers.{layer_i}.attention.wq.weight"]	),
          f"model.layers.{layer_i}.self_attn.k_proj.weight": permute(
              loaded[f"layers.{layer_i}.attention.wk.weight"]	),
          f"model.layers.{layer_i}.self_attn.v_proj.weight": loaded[f"layers.{layer_i}.attention.wv.weight"],
          f"model.layers.{layer_i}.self_attn.o_proj.weight": loaded[f"layers.{layer_i}.attention.wo.weight"],
          f"model.layers.{layer_i}.mlp.gate_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w1.weight"],
          f"model.layers.{layer_i}.mlp.down_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w2.weight"],
          f"model.layers.{layer_i}.mlp.up_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w3.weight"],
          f"model.layers.{layer_i}.input_layernorm.weight": loaded[f"layers.{layer_i}.attention_norm.weight"],
          f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[f"layers.{layer_i}.ffn_norm.weight"],
      }
    else:
      # Sharded
      # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
      # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
      # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

      _lowerCAmelCase						:       str  			=     {
          f"model.layers.{layer_i}.input_layernorm.weight": loaded[0][
              f"layers.{layer_i}.attention_norm.weight"
          ].clone(),
          f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[0][
              f"layers.{layer_i}.ffn_norm.weight"
          ].clone(),
      }
      _lowerCAmelCase						:       Any  			=     permute(
          torch.cat(
              [
                  loaded[i][f"layers.{layer_i}.attention.wq.weight"].view(_lowerCamelCase			,_lowerCamelCase			,_lowerCamelCase	)
                  for i in range(_lowerCamelCase	)
              ]			,dim=0			,).reshape(_lowerCamelCase			,_lowerCamelCase	)	)
      _lowerCAmelCase						:       List[str]  			=     permute(
          torch.cat(
              [
                  loaded[i][f"layers.{layer_i}.attention.wk.weight"].view(
                      _lowerCamelCase			,_lowerCamelCase			,_lowerCamelCase	)
                  for i in range(_lowerCamelCase	)
              ]			,dim=0			,).reshape(_lowerCamelCase			,_lowerCamelCase	)			,_lowerCamelCase			,_lowerCamelCase			,_lowerCamelCase			,)
      _lowerCAmelCase						:       List[Any]  			=     torch.cat(
          [
              loaded[i][f"layers.{layer_i}.attention.wv.weight"].view(
                  _lowerCamelCase			,_lowerCamelCase			,_lowerCamelCase	)
              for i in range(_lowerCamelCase	)
          ]			,dim=0			,).reshape(_lowerCamelCase			,_lowerCamelCase	)

      _lowerCAmelCase						:       List[Any]  			=     torch.cat(
          [loaded[i][f"layers.{layer_i}.attention.wo.weight"] for i in range(_lowerCamelCase	)]			,dim=1	)
      _lowerCAmelCase						:       Union[str, Any]  			=     torch.cat(
          [loaded[i][f"layers.{layer_i}.feed_forward.w1.weight"] for i in range(_lowerCamelCase	)]			,dim=0	)
      _lowerCAmelCase						:       Tuple  			=     torch.cat(
          [loaded[i][f"layers.{layer_i}.feed_forward.w2.weight"] for i in range(_lowerCamelCase	)]			,dim=1	)
      _lowerCAmelCase						:       Dict  			=     torch.cat(
          [loaded[i][f"layers.{layer_i}.feed_forward.w3.weight"] for i in range(_lowerCamelCase	)]			,dim=0	)

    _lowerCAmelCase						:       Optional[Any]  			=     inv_freq
    for k, v in state_dict.items():
      _lowerCAmelCase						:       Tuple  			=     filename
      param_count += v.numel()
    torch.save(_lowerCamelCase			,os.path.join(_lowerCamelCase			,_lowerCamelCase	)	)

  _lowerCAmelCase						:       Optional[Any]  			=     f"pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"
  if model_size == "7B":
    # Unsharded
    _lowerCAmelCase						:       Any  			=     {
        """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
        """model.norm.weight""": loaded["""norm.weight"""],
        """lm_head.weight""": loaded["""output.weight"""],
    }
  else:
    _lowerCAmelCase						:       str  			=     {
        """model.norm.weight""": loaded[0]["""norm.weight"""],
        """model.embed_tokens.weight""": torch.cat(
            [loaded[i]["""tok_embeddings.weight"""] for i in range(_lowerCamelCase	)]			,dim=1	),
        """lm_head.weight""": torch.cat([loaded[i]["""output.weight"""] for i in range(_lowerCamelCase	)]			,dim=0	),
    }

  for k, v in state_dict.items():
    _lowerCAmelCase						:       Tuple  			=     filename
    param_count += v.numel()
  torch.save(_lowerCamelCase			,os.path.join(_lowerCamelCase			,_lowerCamelCase	)	)

  # Write configs
  _lowerCAmelCase						:       Optional[Any]  			=     {"""total_size""": param_count * 2}
  write_json(_lowerCamelCase			,os.path.join(_lowerCamelCase			,"""pytorch_model.bin.index.json"""	)	)
  _lowerCAmelCase						:       Any  			=     params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
  _lowerCAmelCase						:       Optional[Any]  			=     params["""multiple_of"""] if """multiple_of""" in params else 256
  _lowerCAmelCase						:       List[str]  			=     LlamaConfig(
      hidden_size=_lowerCamelCase			,intermediate_size=compute_intermediate_size(_lowerCamelCase			,_lowerCamelCase			,_lowerCamelCase	)			,num_attention_heads=params["""n_heads"""]			,num_hidden_layers=params["""n_layers"""]			,rms_norm_eps=params["""norm_eps"""]			,num_key_value_heads=_lowerCamelCase			,)
  config.save_pretrained(_lowerCamelCase	)

  # Make space so we can load the model properly now.
  del state_dict
  del loaded
  gc.collect()

  print("""Loading the checkpoint in a Llama model."""	)
  _lowerCAmelCase						:       List[Any]  			=     LlamaForCausalLM.from_pretrained(_lowerCamelCase			,torch_dtype=torch.floataa			,low_cpu_mem_usage=_lowerCamelCase	)
  # Avoid saving this as part of the config.
  del model.config._name_or_path

  print("""Saving in the Transformers format."""	)
  model.save_pretrained(_lowerCamelCase			,safe_serialization=_lowerCamelCase	)
  shutil.rmtree(_lowerCamelCase	)





def 							SCREAMING_SNAKE_CASE     (			_lowerCamelCase						:       int			,_lowerCamelCase						:       Optional[int]	)       ->      Tuple:
  # Initialize the tokenizer based on the `spm` model
  _lowerCAmelCase						:       Optional[int]  			=     LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
  print(f"Saving a {tokenizer_class.__name__} to {tokenizer_path}."	)
  _lowerCAmelCase						:       Optional[Any]  			=     tokenizer_class(_lowerCamelCase	)
  tokenizer.save_pretrained(_lowerCamelCase	)





def 							SCREAMING_SNAKE_CASE     (			)       ->      str:
  _lowerCAmelCase						:       List[Any]  			=     argparse.ArgumentParser()
  parser.add_argument(
      """--input_dir"""			,help="""Location of LLaMA weights, which contains tokenizer.model and model folders"""			,)
  parser.add_argument(
      """--model_size"""			,choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""]			,)
  parser.add_argument(
      """--output_dir"""			,help="""Location to write HF model and tokenizer"""			,)
  parser.add_argument("""--safe_serialization"""			,type=_lowerCamelCase			,help="""Whether or not to save using `safetensors`."""	)
  _lowerCAmelCase						:       Any  			=     parser.parse_args()
  if args.model_size != "tokenizer_only":
    write_model(
        model_path=args.output_dir			,input_base_path=os.path.join(args.input_dir			,args.model_size	)			,model_size=args.model_size			,safe_serialization=args.safe_serialization			,)
  _lowerCAmelCase						:       List[Any]  			=     os.path.join(args.input_dir			,"""tokenizer.model"""	)
  write_tokenizer(args.output_dir			,_lowerCamelCase	)


if __name__ == "__main__":
     main()





"""simple docstring"""




from math import ceil





def 							SCREAMING_SNAKE_CASE     (			_lowerCamelCase						:       Union[str, Any]			,_lowerCamelCase						:       Union[str, Any]	)       ->      int:
  _lowerCAmelCase						:       Dict  			=     list(range(0			,_lowerCamelCase	)	)

  _lowerCAmelCase						:       Tuple  			=     [item for sublist in list(device_map.values()	) for item in sublist]

  # Duplicate check
  _lowerCAmelCase						:       Union[str, Any]  			=     []
  for i in device_map_blocks:
    if device_map_blocks.count(_lowerCamelCase	) > 1 and i not in duplicate_blocks:
      duplicate_blocks.append(_lowerCamelCase	)
    # Missing blocks
  _lowerCAmelCase						:       int  			=     [i for i in blocks if i not in device_map_blocks]
  _lowerCAmelCase						:       List[Any]  			=     [i for i in device_map_blocks if i not in blocks]

  if len(_lowerCamelCase	) != 0:
    raise ValueError(
        """Duplicate attention blocks specified in device_map. Attention blocks must be specified to one device."""
        """ These attention blocks were specified more than once: """ + str(_lowerCamelCase	)	)
  if len(_lowerCamelCase	) != 0:
    raise ValueError(
        """There are attention blocks for this model that are not specified in the device_map. Add these attention """
        """blocks to a device on the device_map: """ + str(_lowerCamelCase	)	)
  if len(_lowerCamelCase	) != 0:
    raise ValueError(
        """The device_map contains more attention blocks than this model has. Remove these from the device_map:"""
        + str(_lowerCamelCase	)	)






def 							SCREAMING_SNAKE_CASE     (			_lowerCamelCase						:       int			,_lowerCamelCase						:       Tuple	)       ->      str:
  _lowerCAmelCase						:       Optional[Any]  			=     list(range(_lowerCamelCase	)	)
  _lowerCAmelCase						:       Optional[Any]  			=     int(ceil(n_layers / len(_lowerCamelCase	)	)	)
  _lowerCAmelCase						:       Optional[int]  			=     [layers[i : i + n_blocks] for i in range(0			,_lowerCamelCase			,_lowerCamelCase	)]

  return dict(zip(_lowerCamelCase			,_lowerCamelCase	)	)




