


import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
  from transformers import LlamaTokenizerFast
except ImportError as e:
  warnings.warn(e)
  warnings.warn(
      '''The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'''
  )
  UpperCAmelCase__   	=							None
UpperCAmelCase__   	=							{
    '''7B''': 1_1008,
    '''13B''': 1_3824,
    '''30B''': 1_7920,
    '''65B''': 2_2016,
    '''70B''': 2_8672,
}
UpperCAmelCase__   	=							{
    '''7B''': 1,
    '''7Bf''': 1,
    '''13B''': 2,
    '''13Bf''': 2,
    '''30B''': 4,
    '''65B''': 8,
    '''70B''': 8,
    '''70Bf''': 8,
}
def 				UpperCAmelCase_					(							__snake_case     ,						__snake_case=1     ,						__snake_case=256						)     ->			Optional[int]:







 """simple docstring"""


 return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3						)						) + multiple_of - 1) // multiple_of)
def 				UpperCAmelCase_					(							__snake_case						)     ->			str:







 """simple docstring"""


 with open(__snake_case     ,						'''r'''						) as f:
  return json.load(__snake_case						)
def 				UpperCAmelCase_					(							__snake_case     ,						__snake_case						)     ->			List[str]:







 """simple docstring"""


 with open(__snake_case     ,						'''w'''						) as f:
  json.dump(__snake_case     ,						__snake_case						)
def 				UpperCAmelCase_					(							__snake_case     ,						__snake_case     ,						__snake_case     ,						__snake_case=True						)     ->			Optional[int]:







 """simple docstring"""


 os.makedirs(__snake_case     ,						exist_ok=__snake_case						)
 _lowercase							      =os.path.join(__snake_case     ,						'''tmp'''						)
 os.makedirs(__snake_case     ,						exist_ok=__snake_case						)

 _lowercase							      =read_json(os.path.join(__snake_case     ,						'''params.json'''						)						)
 _lowercase							      =NUM_SHARDS[model_size]
 _lowercase							      =params['''n_layers''']
 _lowercase							      =params['''n_heads''']
 _lowercase							      =n_heads // num_shards
 _lowercase							      =params['''dim''']
 _lowercase							      =dim // n_heads
 _lowercase							      =1_00_00.0
 _lowercase							      =1.0 / (base ** (torch.arange(0     ,						__snake_case     ,						2						).float() / dims_per_head))

 if "n_kv_heads" in params:
  _lowercase							      =params['''n_kv_heads''']  # for GQA / MQA
  _lowercase							      =n_heads_per_shard // num_key_value_heads
  _lowercase							      =dim // num_key_value_heads
 else:  # compatibility with other checkpoints
  _lowercase							      =n_heads
  _lowercase							      =n_heads_per_shard
  _lowercase							      =dim

 # permute for sliced rotary
 def permute(__snake_case     ,						__snake_case=n_heads     ,						__snake_case=dim     ,						__snake_case=dim						):
  return w.view(__snake_case     ,						dima // n_heads // 2     ,						2     ,						__snake_case						).transpose(1     ,						2						).reshape(__snake_case     ,						__snake_case						)

 print(F"Fetching all parameters from the checkpoint at {input_base_path}."						)
 # Load weights
 if model_size == "7B":
  # Not sharded
  # (The sharded implementation would also work, but this is simpler.)
  _lowercase							      =torch.load(os.path.join(__snake_case     ,						'''consolidated.00.pth'''						)     ,						map_location='''cpu'''						)
 else:
  # Sharded
  _lowercase							      =[
      torch.load(os.path.join(__snake_case     ,						F"consolidated.{i:02d}.pth"						)     ,						map_location='''cpu'''						)
      for i in range(__snake_case						)
  ]
 _lowercase							      =0
 _lowercase							      ={'''weight_map''': {}}
 for layer_i in range(__snake_case						):
  _lowercase							      =F"pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"
  if model_size == "7B":
   # Unsharded
   _lowercase							      ={
       F"model.layers.{layer_i}.self_attn.q_proj.weight": permute(
           loaded[F"layers.{layer_i}.attention.wq.weight"]						),
       F"model.layers.{layer_i}.self_attn.k_proj.weight": permute(
           loaded[F"layers.{layer_i}.attention.wk.weight"]						),
       F"model.layers.{layer_i}.self_attn.v_proj.weight": loaded[F"layers.{layer_i}.attention.wv.weight"],
       F"model.layers.{layer_i}.self_attn.o_proj.weight": loaded[F"layers.{layer_i}.attention.wo.weight"],
       F"model.layers.{layer_i}.mlp.gate_proj.weight": loaded[F"layers.{layer_i}.feed_forward.w1.weight"],
       F"model.layers.{layer_i}.mlp.down_proj.weight": loaded[F"layers.{layer_i}.feed_forward.w2.weight"],
       F"model.layers.{layer_i}.mlp.up_proj.weight": loaded[F"layers.{layer_i}.feed_forward.w3.weight"],
       F"model.layers.{layer_i}.input_layernorm.weight": loaded[F"layers.{layer_i}.attention_norm.weight"],
       F"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[F"layers.{layer_i}.ffn_norm.weight"],
   }
  else:
   # Sharded
   # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
   # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
   # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

   _lowercase							      ={
       F"model.layers.{layer_i}.input_layernorm.weight": loaded[0][
           F"layers.{layer_i}.attention_norm.weight"
       ].clone(),
       F"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[0][
           F"layers.{layer_i}.ffn_norm.weight"
       ].clone(),
   }
   _lowercase							      =permute(
       torch.cat(
           [
               loaded[i][F"layers.{layer_i}.attention.wq.weight"].view(__snake_case     ,						__snake_case     ,						__snake_case						)
               for i in range(__snake_case						)
           ]     ,						dim=0     ,						).reshape(__snake_case     ,						__snake_case						)						)
   _lowercase							      =permute(
       torch.cat(
           [
               loaded[i][F"layers.{layer_i}.attention.wk.weight"].view(
                   __snake_case     ,						__snake_case     ,						__snake_case						)
               for i in range(__snake_case						)
           ]     ,						dim=0     ,						).reshape(__snake_case     ,						__snake_case						)     ,						__snake_case     ,						__snake_case     ,						__snake_case     ,						)
   _lowercase							      =torch.cat(
       [
           loaded[i][F"layers.{layer_i}.attention.wv.weight"].view(
               __snake_case     ,						__snake_case     ,						__snake_case						)
           for i in range(__snake_case						)
       ]     ,						dim=0     ,						).reshape(__snake_case     ,						__snake_case						)

   _lowercase							      =torch.cat(
       [loaded[i][F"layers.{layer_i}.attention.wo.weight"] for i in range(__snake_case						)]     ,						dim=1						)
   _lowercase							      =torch.cat(
       [loaded[i][F"layers.{layer_i}.feed_forward.w1.weight"] for i in range(__snake_case						)]     ,						dim=0						)
   _lowercase							      =torch.cat(
       [loaded[i][F"layers.{layer_i}.feed_forward.w2.weight"] for i in range(__snake_case						)]     ,						dim=1						)
   _lowercase							      =torch.cat(
       [loaded[i][F"layers.{layer_i}.feed_forward.w3.weight"] for i in range(__snake_case						)]     ,						dim=0						)

  _lowercase							      =inv_freq
  for k, v in state_dict.items():
   _lowercase							      =filename
   param_count += v.numel()
  torch.save(__snake_case     ,						os.path.join(__snake_case     ,						__snake_case						)						)

 _lowercase							      =F"pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"
 if model_size == "7B":
  # Unsharded
  _lowercase							      ={
      '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
      '''model.norm.weight''': loaded['''norm.weight'''],
      '''lm_head.weight''': loaded['''output.weight'''],
  }
 else:
  _lowercase							      ={
      '''model.norm.weight''': loaded[0]['''norm.weight'''],
      '''model.embed_tokens.weight''': torch.cat(
          [loaded[i]['''tok_embeddings.weight'''] for i in range(__snake_case						)]     ,						dim=1						),
      '''lm_head.weight''': torch.cat([loaded[i]['''output.weight'''] for i in range(__snake_case						)]     ,						dim=0						),
  }

 for k, v in state_dict.items():
  _lowercase							      =filename
  param_count += v.numel()
 torch.save(__snake_case     ,						os.path.join(__snake_case     ,						__snake_case						)						)

 # Write configs
 _lowercase							      ={'''total_size''': param_count * 2}
 write_json(__snake_case     ,						os.path.join(__snake_case     ,						'''pytorch_model.bin.index.json'''						)						)
 _lowercase							      =params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
 _lowercase							      =params['''multiple_of'''] if '''multiple_of''' in params else 256
 _lowercase							      =LlamaConfig(
     hidden_size=__snake_case     ,						intermediate_size=compute_intermediate_size(__snake_case     ,						__snake_case     ,						__snake_case						)     ,						num_attention_heads=params['''n_heads''']     ,						num_hidden_layers=params['''n_layers''']     ,						rms_norm_eps=params['''norm_eps''']     ,						num_key_value_heads=__snake_case     ,						)
 config.save_pretrained(__snake_case						)

 # Make space so we can load the model properly now.
 del state_dict
 del loaded
 gc.collect()

 print('''Loading the checkpoint in a Llama model.'''						)
 _lowercase							      =LlamaForCausalLM.from_pretrained(__snake_case     ,						torch_dtype=torch.floataa     ,						low_cpu_mem_usage=__snake_case						)
 # Avoid saving this as part of the config.
 del model.config._name_or_path

 print('''Saving in the Transformers format.'''						)
 model.save_pretrained(__snake_case     ,						safe_serialization=__snake_case						)
 shutil.rmtree(__snake_case						)
def 				UpperCAmelCase_					(							__snake_case     ,						__snake_case						)     ->			List[str]:







 """simple docstring"""


 _lowercase							      =LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
 print(F"Saving a {tokenizer_class.__name__} to {tokenizer_path}."						)
 _lowercase							      =tokenizer_class(__snake_case						)
 tokenizer.save_pretrained(__snake_case						)
def 				UpperCAmelCase_					(							)     ->			Tuple:







 """simple docstring"""


 _lowercase							      =argparse.ArgumentParser()
 parser.add_argument(
     '''--input_dir'''     ,						help='''Location of LLaMA weights, which contains tokenizer.model and model folders'''     ,						)
 parser.add_argument(
     '''--model_size'''     ,						choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only''']     ,						)
 parser.add_argument(
     '''--output_dir'''     ,						help='''Location to write HF model and tokenizer'''     ,						)
 parser.add_argument('''--safe_serialization'''     ,						type=__snake_case     ,						help='''Whether or not to save using `safetensors`.'''						)
 _lowercase							      =parser.parse_args()
 if args.model_size != "tokenizer_only":
  write_model(
      model_path=args.output_dir     ,						input_base_path=os.path.join(args.input_dir     ,						args.model_size						)     ,						model_size=args.model_size     ,						safe_serialization=args.safe_serialization     ,						)
 _lowercase							      =os.path.join(args.input_dir     ,						'''tokenizer.model'''						)
 write_tokenizer(args.output_dir     ,						__snake_case						)


if __name__ == "__main__":
  main()





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


UpperCAmelCase__   	=							{
    '''configuration_xlm''': ['''XLM_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''XLMConfig''', '''XLMOnnxConfig'''],
    '''tokenization_xlm''': ['''XLMTokenizer'''],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase__   	=							[
      '''XLM_PRETRAINED_MODEL_ARCHIVE_LIST''',
      '''XLMForMultipleChoice''',
      '''XLMForQuestionAnswering''',
      '''XLMForQuestionAnsweringSimple''',
      '''XLMForSequenceClassification''',
      '''XLMForTokenClassification''',
      '''XLMModel''',
      '''XLMPreTrainedModel''',
      '''XLMWithLMHeadModel''',
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase__   	=							[
      '''TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST''',
      '''TFXLMForMultipleChoice''',
      '''TFXLMForQuestionAnsweringSimple''',
      '''TFXLMForSequenceClassification''',
      '''TFXLMForTokenClassification''',
      '''TFXLMMainLayer''',
      '''TFXLMModel''',
      '''TFXLMPreTrainedModel''',
      '''TFXLMWithLMHeadModel''',
  ]


if TYPE_CHECKING:
  from .configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig, XLMOnnxConfig
  from .tokenization_xlm import XLMTokenizer

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_xlm import (
        XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
        XLMForMultipleChoice,
        XLMForQuestionAnswering,
        XLMForQuestionAnsweringSimple,
        XLMForSequenceClassification,
        XLMForTokenClassification,
        XLMModel,
        XLMPreTrainedModel,
        XLMWithLMHeadModel,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_xlm import (
        TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFXLMForMultipleChoice,
        TFXLMForQuestionAnsweringSimple,
        TFXLMForSequenceClassification,
        TFXLMForTokenClassification,
        TFXLMMainLayer,
        TFXLMModel,
        TFXLMPreTrainedModel,
        TFXLMWithLMHeadModel,
    )

else:
  import sys

  UpperCAmelCase__   	=							_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)

