"""simple docstring"""





import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
       from transformers import LlamaTokenizerFast
except ImportError as e:
       warnings.warn(e)
       warnings.warn(
           "The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"
       )
       _UpperCamelCase   :						Tuple      =    None
_UpperCamelCase   :						str      =    {
    "7B": 1_10_08,
    "13B": 1_38_24,
    "30B": 1_79_20,
    "65B": 2_20_16,
    "70B": 2_86_72,
}
_UpperCamelCase   :						int      =    {
    "7B": 1,
    "7Bf": 1,
    "13B": 2,
    "13Bf": 2,
    "30B": 4,
    "65B": 8,
    "70B": 8,
    "70Bf": 8,
}





def 				a_     (      _lowerCAmelCase  : Any   , _lowerCAmelCase  : Tuple=1   , _lowerCAmelCase  : Union[str, Any]=256     ):





       '''simple docstring'''

       return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3     )     ) + multiple_of - 1) // multiple_of)





def 				a_     (      _lowerCAmelCase  : int     ):





       '''simple docstring'''

       with open(_lowerCAmelCase   , 'r'     ) as f:
              return json.load(_lowerCAmelCase     )





def 				a_     (      _lowerCAmelCase  : Any   , _lowerCAmelCase  : List[Any]     ):





       '''simple docstring'''

       with open(_lowerCAmelCase   , 'w'     ) as f:
              json.dump(_lowerCAmelCase   , _lowerCAmelCase     )





def 				a_     (      _lowerCAmelCase  : List[Any]   , _lowerCAmelCase  : Union[str, Any]   , _lowerCAmelCase  : Dict   , _lowerCAmelCase  : Dict=True     ):





       '''simple docstring'''

       os.makedirs(_lowerCAmelCase   , exist_ok=_lowerCAmelCase     )
       lowercase__			:		str       =	os.path.join(_lowerCAmelCase   , 'tmp'     )
       os.makedirs(_lowerCAmelCase   , exist_ok=_lowerCAmelCase     )

       lowercase__			:		List[str]       =	read_json(os.path.join(_lowerCAmelCase   , 'params.json'     )     )
       lowercase__			:		Any       =	NUM_SHARDS[model_size]
       lowercase__			:		str       =	params['n_layers']
       lowercase__			:		List[Any]       =	params['n_heads']
       lowercase__			:		Tuple       =	n_heads // num_shards
       lowercase__			:		Tuple       =	params['dim']
       lowercase__			:		List[str]       =	dim // n_heads
       lowercase__			:		Dict       =	1_0_0_0_0.0
       lowercase__			:		Any       =	1.0 / (base ** (torch.arange(0   , _lowerCAmelCase   , 2     ).float() / dims_per_head))

       if "n_kv_heads" in params:
              lowercase__			:		Any       =	params['n_kv_heads']  # for GQA / MQA
              lowercase__			:		Tuple       =	n_heads_per_shard // num_key_value_heads
              lowercase__			:		Optional[Any]       =	dim // num_key_value_heads
       else:  # compatibility with other checkpoints
              lowercase__			:		Dict       =	n_heads
              lowercase__			:		Tuple       =	n_heads_per_shard
              lowercase__			:		int       =	dim

       # permute for sliced rotary
       def permute(_lowerCAmelCase  : Optional[int]   , _lowerCAmelCase  : Dict=n_heads   , _lowerCAmelCase  : Any=dim   , _lowerCAmelCase  : List[Any]=dim     ):
              return w.view(_lowerCAmelCase   , dima // n_heads // 2   , 2   , _lowerCAmelCase     ).transpose(1   , 2     ).reshape(_lowerCAmelCase   , _lowerCAmelCase     )

       print(f"""Fetching all parameters from the checkpoint at {input_base_path}."""     )
       # Load weights
       if model_size == "7B":
              # Not sharded
              # (The sharded implementation would also work, but this is simpler.)
              lowercase__			:		Tuple       =	torch.load(os.path.join(_lowerCAmelCase   , 'consolidated.00.pth'     )   , map_location='cpu'     )
       else:
              # Sharded
              lowercase__			:		int       =	[
                  torch.load(os.path.join(_lowerCAmelCase   , f"""consolidated.{i:02d}.pth"""     )   , map_location='cpu'     )
                  for i in range(_lowerCAmelCase     )
              ]
       lowercase__			:		List[Any]       =	0
       lowercase__			:		Dict       =	{'weight_map': {}}
       for layer_i in range(_lowerCAmelCase     ):
              lowercase__			:		List[Any]       =	f"""pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"""
              if model_size == "7B":
                     # Unsharded
                     lowercase__			:		Any       =	{
                         f"""model.layers.{layer_i}.self_attn.q_proj.weight""": permute(
                             loaded[f"""layers.{layer_i}.attention.wq.weight"""]     ),
                         f"""model.layers.{layer_i}.self_attn.k_proj.weight""": permute(
                             loaded[f"""layers.{layer_i}.attention.wk.weight"""]     ),
                         f"""model.layers.{layer_i}.self_attn.v_proj.weight""": loaded[f"""layers.{layer_i}.attention.wv.weight"""],
                         f"""model.layers.{layer_i}.self_attn.o_proj.weight""": loaded[f"""layers.{layer_i}.attention.wo.weight"""],
                         f"""model.layers.{layer_i}.mlp.gate_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w1.weight"""],
                         f"""model.layers.{layer_i}.mlp.down_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w2.weight"""],
                         f"""model.layers.{layer_i}.mlp.up_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w3.weight"""],
                         f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[f"""layers.{layer_i}.attention_norm.weight"""],
                         f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[f"""layers.{layer_i}.ffn_norm.weight"""],
                     }
              else:
                     # Sharded
                     # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
                     # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
                     # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

                     lowercase__			:		Optional[int]       =	{
                         f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[0][
                             f"""layers.{layer_i}.attention_norm.weight"""
                         ].clone(),
                         f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[0][
                             f"""layers.{layer_i}.ffn_norm.weight"""
                         ].clone(),
                     }
                     lowercase__			:		Optional[Any]       =	permute(
                         torch.cat(
                             [
                                 loaded[i][f"""layers.{layer_i}.attention.wq.weight"""].view(_lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase     )
                                 for i in range(_lowerCAmelCase     )
                             ]   , dim=0   , ).reshape(_lowerCAmelCase   , _lowerCAmelCase     )     )
                     lowercase__			:		List[str]       =	permute(
                         torch.cat(
                             [
                                 loaded[i][f"""layers.{layer_i}.attention.wk.weight"""].view(
                                     _lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase     )
                                 for i in range(_lowerCAmelCase     )
                             ]   , dim=0   , ).reshape(_lowerCAmelCase   , _lowerCAmelCase     )   , _lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase   , )
                     lowercase__			:		Optional[int]       =	torch.cat(
                         [
                             loaded[i][f"""layers.{layer_i}.attention.wv.weight"""].view(
                                 _lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase     )
                             for i in range(_lowerCAmelCase     )
                         ]   , dim=0   , ).reshape(_lowerCAmelCase   , _lowerCAmelCase     )

                     lowercase__			:		Tuple       =	torch.cat(
                         [loaded[i][f"""layers.{layer_i}.attention.wo.weight"""] for i in range(_lowerCAmelCase     )]   , dim=1     )
                     lowercase__			:		Union[str, Any]       =	torch.cat(
                         [loaded[i][f"""layers.{layer_i}.feed_forward.w1.weight"""] for i in range(_lowerCAmelCase     )]   , dim=0     )
                     lowercase__			:		Optional[int]       =	torch.cat(
                         [loaded[i][f"""layers.{layer_i}.feed_forward.w2.weight"""] for i in range(_lowerCAmelCase     )]   , dim=1     )
                     lowercase__			:		Any       =	torch.cat(
                         [loaded[i][f"""layers.{layer_i}.feed_forward.w3.weight"""] for i in range(_lowerCAmelCase     )]   , dim=0     )

              lowercase__			:		Optional[Any]       =	inv_freq
              for k, v in state_dict.items():
                     lowercase__			:		Tuple       =	filename
                     param_count += v.numel()
              torch.save(_lowerCAmelCase   , os.path.join(_lowerCAmelCase   , _lowerCAmelCase     )     )

       lowercase__			:		Dict       =	f"""pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"""
       if model_size == "7B":
              # Unsharded
              lowercase__			:		Tuple       =	{
                  'model.embed_tokens.weight': loaded['tok_embeddings.weight'],
                  'model.norm.weight': loaded['norm.weight'],
                  'lm_head.weight': loaded['output.weight'],
              }
       else:
              lowercase__			:		int       =	{
                  'model.norm.weight': loaded[0]['norm.weight'],
                  'model.embed_tokens.weight': torch.cat(
                      [loaded[i]['tok_embeddings.weight'] for i in range(_lowerCAmelCase     )]   , dim=1     ),
                  'lm_head.weight': torch.cat([loaded[i]['output.weight'] for i in range(_lowerCAmelCase     )]   , dim=0     ),
              }

       for k, v in state_dict.items():
              lowercase__			:		Optional[Any]       =	filename
              param_count += v.numel()
       torch.save(_lowerCAmelCase   , os.path.join(_lowerCAmelCase   , _lowerCAmelCase     )     )

       # Write configs
       lowercase__			:		int       =	{'total_size': param_count * 2}
       write_json(_lowerCAmelCase   , os.path.join(_lowerCAmelCase   , 'pytorch_model.bin.index.json'     )     )
       lowercase__			:		str       =	params['ffn_dim_multiplier'] if 'ffn_dim_multiplier' in params else 1
       lowercase__			:		Optional[Any]       =	params['multiple_of'] if 'multiple_of' in params else 256
       lowercase__			:		Tuple       =	LlamaConfig(
           hidden_size=_lowerCAmelCase   , intermediate_size=compute_intermediate_size(_lowerCAmelCase   , _lowerCAmelCase   , _lowerCAmelCase     )   , num_attention_heads=params['n_heads']   , num_hidden_layers=params['n_layers']   , rms_norm_eps=params['norm_eps']   , num_key_value_heads=_lowerCAmelCase   , )
       config.save_pretrained(_lowerCAmelCase     )

       # Make space so we can load the model properly now.
       del state_dict
       del loaded
       gc.collect()

       print('Loading the checkpoint in a Llama model.'     )
       lowercase__			:		Dict       =	LlamaForCausalLM.from_pretrained(_lowerCAmelCase   , torch_dtype=torch.floataa   , low_cpu_mem_usage=_lowerCAmelCase     )
       # Avoid saving this as part of the config.
       del model.config._name_or_path

       print('Saving in the Transformers format.'     )
       model.save_pretrained(_lowerCAmelCase   , safe_serialization=_lowerCAmelCase     )
       shutil.rmtree(_lowerCAmelCase     )





def 				a_     (      _lowerCAmelCase  : Optional[Any]   , _lowerCAmelCase  : Tuple     ):





       '''simple docstring'''

       lowercase__			:		Optional[Any]       =	LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
       print(f"""Saving a {tokenizer_class.__name__} to {tokenizer_path}."""     )
       lowercase__			:		Tuple       =	tokenizer_class(_lowerCAmelCase     )
       tokenizer.save_pretrained(_lowerCAmelCase     )





def 				a_     (      ):





       '''simple docstring'''

       lowercase__			:		Dict       =	argparse.ArgumentParser()
       parser.add_argument(
           '--input_dir'   , help='Location of LLaMA weights, which contains tokenizer.model and model folders'   , )
       parser.add_argument(
           '--model_size'   , choices=['7B', '7Bf', '13B', '13Bf', '30B', '65B', '70B', '70Bf', 'tokenizer_only']   , )
       parser.add_argument(
           '--output_dir'   , help='Location to write HF model and tokenizer'   , )
       parser.add_argument('--safe_serialization'   , type=_lowerCAmelCase   , help='Whether or not to save using `safetensors`.'     )
       lowercase__			:		str       =	parser.parse_args()
       if args.model_size != "tokenizer_only":
              write_model(
                  model_path=args.output_dir   , input_base_path=os.path.join(args.input_dir   , args.model_size     )   , model_size=args.model_size   , safe_serialization=args.safe_serialization   , )
       lowercase__			:		str       =	os.path.join(args.input_dir   , 'tokenizer.model'     )
       write_tokenizer(args.output_dir   , _lowerCAmelCase     )


if __name__ == "__main__":
       main()






'''simple docstring'''


import math

class  SCREAMING_SNAKE_CASE :






    """simple docstring"""





    def __init__(  self      :					Union[str, Any]  ,						UpperCamelCase__      :					Optional[Any]=0 ):  # a graph with Node 0,1,...,N-1


       """simple docstring"""







       UpperCamelCase					=  n
       UpperCamelCase					=  [
           [math.inf for j in range(0  ,						UpperCamelCase__ )] for i in range(0  ,						UpperCamelCase__ )
       ]  # adjacency matrix for weight
       UpperCamelCase					=  [
           [math.inf for j in range(0  ,						UpperCamelCase__ )] for i in range(0  ,						UpperCamelCase__ )
       ]  # dp[i][j] stores minimum distance from i to j






    def  A       (  self      :					Optional[Any]  ,						UpperCamelCase__      :					Union[str, Any]  ,						UpperCamelCase__      :					Union[str, Any]  ,						UpperCamelCase__      :					Tuple ):


       """simple docstring"""







       UpperCamelCase					=  w






    def  A       (  self      :					str ):


       """simple docstring"""







       for k in range(0  ,						self.n ):
          for i in range(0  ,						self.n ):
             for j in range(0  ,						self.n ):
                UpperCamelCase					=  min(self.dp[i][j]  ,						self.dp[i][k] + self.dp[k][j] )






    def  A       (  self      :					Optional[Any]  ,						UpperCamelCase__      :					str  ,						UpperCamelCase__      :					List[Any] ):


       """simple docstring"""







       return self.dp[u][v]


if __name__ == "__main__":
   _lowerCamelCase	:       List[str]      	=	Graph(5)
   graph.add_edge(0, 2, 9)
   graph.add_edge(0, 4, 10)
   graph.add_edge(1, 3, 5)
   graph.add_edge(2, 3, 7)
   graph.add_edge(3, 0, 10)
   graph.add_edge(3, 1, 2)
   graph.add_edge(3, 2, 1)
   graph.add_edge(3, 4, 6)
   graph.add_edge(4, 1, 3)
   graph.add_edge(4, 2, 4)
   graph.add_edge(4, 3, 9)
   graph.floyd_warshall()
   graph.show_min(1, 4)
   graph.show_min(0, 3)

