




'''simple docstring'''

import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
							from transformers import LlamaTokenizerFast
except ImportError as e:
							warnings.warn(e)
							warnings.warn(
							    'The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'
							)
							a  :				Dict									=  None
a  :				Optional[int]									=  {
    '7B': 1_1008,
    '13B': 1_3824,
    '30B': 1_7920,
    '65B': 2_2016,
    '70B': 2_8672,
}
a  :				Union[str, Any]									=  {
    '7B': 1,
    '7Bf': 1,
    '13B': 2,
    '13Bf': 2,
    '30B': 4,
    '65B': 8,
    '70B': 8,
    '70Bf': 8,
}


def 		__magic_name__    (	__UpperCAmelCase,      __UpperCAmelCase=1,      __UpperCAmelCase=256	)		->					Dict:

					'''simple docstring'''

					return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3	)	) + multiple_of - 1) // multiple_of)


def 		__magic_name__    (	__UpperCAmelCase	)		->					Optional[int]:

					'''simple docstring'''

					with open(__UpperCAmelCase,      '''r'''	) as f:
										return json.load(__UpperCAmelCase	)


def 		__magic_name__    (	__UpperCAmelCase,      __UpperCAmelCase	)		->					Any:

					'''simple docstring'''

					with open(__UpperCAmelCase,      '''w'''	) as f:
										json.dump(__UpperCAmelCase,      __UpperCAmelCase	)


def 		__magic_name__    (	__UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase=True	)		->					Optional[int]:

					'''simple docstring'''

					os.makedirs(__UpperCAmelCase,      exist_ok=__UpperCAmelCase	)
					snake_case_									=				os.path.join(__UpperCAmelCase,      '''tmp'''	)
					os.makedirs(__UpperCAmelCase,      exist_ok=__UpperCAmelCase	)

					snake_case_									=				read_json(os.path.join(__UpperCAmelCase,      '''params.json'''	)	)
					snake_case_									=				NUM_SHARDS[model_size]
					snake_case_									=				params['''n_layers''']
					snake_case_									=				params['''n_heads''']
					snake_case_									=				n_heads // num_shards
					snake_case_									=				params['''dim''']
					snake_case_									=				dim // n_heads
					snake_case_									=				1_0_0_0_0.0
					snake_case_									=				1.0 / (base ** (torch.arange(0,      __UpperCAmelCase,      2	).float() / dims_per_head))

					if "n_kv_heads" in params:
										snake_case_									=				params['''n_kv_heads''']  # for GQA / MQA
										snake_case_									=				n_heads_per_shard // num_key_value_heads
										snake_case_									=				dim // num_key_value_heads
					else:  # compatibility with other checkpoints
										snake_case_									=				n_heads
										snake_case_									=				n_heads_per_shard
										snake_case_									=				dim

					# permute for sliced rotary
					def permute(__UpperCAmelCase,      __UpperCAmelCase=n_heads,      __UpperCAmelCase=dim,      __UpperCAmelCase=dim	):
										return w.view(__UpperCAmelCase,      dima // n_heads // 2,      2,      __UpperCAmelCase	).transpose(1,      2	).reshape(__UpperCAmelCase,      __UpperCAmelCase	)

					print(F"Fetching all parameters from the checkpoint at {input_base_path}."	)
					# Load weights
					if model_size == "7B":
										# Not sharded
										# (The sharded implementation would also work, but this is simpler.)
										snake_case_									=				torch.load(os.path.join(__UpperCAmelCase,      '''consolidated.00.pth'''	),      map_location='''cpu'''	)
					else:
										# Sharded
										snake_case_									=				[
										    torch.load(os.path.join(__UpperCAmelCase,      F"consolidated.{i:02d}.pth"	),      map_location='''cpu'''	)
										    for i in range(__UpperCAmelCase	)
										]
					snake_case_									=				0
					snake_case_									=				{'''weight_map''': {}}
					for layer_i in range(__UpperCAmelCase	):
										snake_case_									=				F"pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"
										if model_size == "7B":
															# Unsharded
															snake_case_									=				{
															    F"model.layers.{layer_i}.self_attn.q_proj.weight": permute(
															        loaded[F"layers.{layer_i}.attention.wq.weight"]	),
															    F"model.layers.{layer_i}.self_attn.k_proj.weight": permute(
															        loaded[F"layers.{layer_i}.attention.wk.weight"]	),
															    F"model.layers.{layer_i}.self_attn.v_proj.weight": loaded[F"layers.{layer_i}.attention.wv.weight"],
															    F"model.layers.{layer_i}.self_attn.o_proj.weight": loaded[F"layers.{layer_i}.attention.wo.weight"],
															    F"model.layers.{layer_i}.mlp.gate_proj.weight": loaded[F"layers.{layer_i}.feed_forward.w1.weight"],
															    F"model.layers.{layer_i}.mlp.down_proj.weight": loaded[F"layers.{layer_i}.feed_forward.w2.weight"],
															    F"model.layers.{layer_i}.mlp.up_proj.weight": loaded[F"layers.{layer_i}.feed_forward.w3.weight"],
															    F"model.layers.{layer_i}.input_layernorm.weight": loaded[F"layers.{layer_i}.attention_norm.weight"],
															    F"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[F"layers.{layer_i}.ffn_norm.weight"],
															}
										else:
															# Sharded
															# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
															# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
															# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

															snake_case_									=				{
															    F"model.layers.{layer_i}.input_layernorm.weight": loaded[0][
															        F"layers.{layer_i}.attention_norm.weight"
															    ].clone(),
															    F"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[0][
															        F"layers.{layer_i}.ffn_norm.weight"
															    ].clone(),
															}
															snake_case_									=				permute(
															    torch.cat(
															        [
															            loaded[i][F"layers.{layer_i}.attention.wq.weight"].view(__UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase	)
															            for i in range(__UpperCAmelCase	)
															        ],      dim=0,      ).reshape(__UpperCAmelCase,      __UpperCAmelCase	)	)
															snake_case_									=				permute(
															    torch.cat(
															        [
															            loaded[i][F"layers.{layer_i}.attention.wk.weight"].view(
															                __UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase	)
															            for i in range(__UpperCAmelCase	)
															        ],      dim=0,      ).reshape(__UpperCAmelCase,      __UpperCAmelCase	),      __UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase,      )
															snake_case_									=				torch.cat(
															    [
															        loaded[i][F"layers.{layer_i}.attention.wv.weight"].view(
															            __UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase	)
															        for i in range(__UpperCAmelCase	)
															    ],      dim=0,      ).reshape(__UpperCAmelCase,      __UpperCAmelCase	)

															snake_case_									=				torch.cat(
															    [loaded[i][F"layers.{layer_i}.attention.wo.weight"] for i in range(__UpperCAmelCase	)],      dim=1	)
															snake_case_									=				torch.cat(
															    [loaded[i][F"layers.{layer_i}.feed_forward.w1.weight"] for i in range(__UpperCAmelCase	)],      dim=0	)
															snake_case_									=				torch.cat(
															    [loaded[i][F"layers.{layer_i}.feed_forward.w2.weight"] for i in range(__UpperCAmelCase	)],      dim=1	)
															snake_case_									=				torch.cat(
															    [loaded[i][F"layers.{layer_i}.feed_forward.w3.weight"] for i in range(__UpperCAmelCase	)],      dim=0	)

										snake_case_									=				inv_freq
										for k, v in state_dict.items():
															snake_case_									=				filename
															param_count += v.numel()
										torch.save(__UpperCAmelCase,      os.path.join(__UpperCAmelCase,      __UpperCAmelCase	)	)

					snake_case_									=				F"pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"
					if model_size == "7B":
										# Unsharded
										snake_case_									=				{
										    '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
										    '''model.norm.weight''': loaded['''norm.weight'''],
										    '''lm_head.weight''': loaded['''output.weight'''],
										}
					else:
										snake_case_									=				{
										    '''model.norm.weight''': loaded[0]['''norm.weight'''],
										    '''model.embed_tokens.weight''': torch.cat(
										        [loaded[i]['''tok_embeddings.weight'''] for i in range(__UpperCAmelCase	)],      dim=1	),
										    '''lm_head.weight''': torch.cat([loaded[i]['''output.weight'''] for i in range(__UpperCAmelCase	)],      dim=0	),
										}

					for k, v in state_dict.items():
										snake_case_									=				filename
										param_count += v.numel()
					torch.save(__UpperCAmelCase,      os.path.join(__UpperCAmelCase,      __UpperCAmelCase	)	)

					# Write configs
					snake_case_									=				{'''total_size''': param_count * 2}
					write_json(__UpperCAmelCase,      os.path.join(__UpperCAmelCase,      '''pytorch_model.bin.index.json'''	)	)
					snake_case_									=				params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
					snake_case_									=				params['''multiple_of'''] if '''multiple_of''' in params else 256
					snake_case_									=				LlamaConfig(
					    hidden_size=__UpperCAmelCase,      intermediate_size=compute_intermediate_size(__UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase	),      num_attention_heads=params['''n_heads'''],      num_hidden_layers=params['''n_layers'''],      rms_norm_eps=params['''norm_eps'''],      num_key_value_heads=__UpperCAmelCase,      )
					config.save_pretrained(__UpperCAmelCase	)

					# Make space so we can load the model properly now.
					del state_dict
					del loaded
					gc.collect()

					print('''Loading the checkpoint in a Llama model.'''	)
					snake_case_									=				LlamaForCausalLM.from_pretrained(__UpperCAmelCase,      torch_dtype=torch.floataa,      low_cpu_mem_usage=__UpperCAmelCase	)
					# Avoid saving this as part of the config.
					del model.config._name_or_path

					print('''Saving in the Transformers format.'''	)
					model.save_pretrained(__UpperCAmelCase,      safe_serialization=__UpperCAmelCase	)
					shutil.rmtree(__UpperCAmelCase	)


def 		__magic_name__    (	__UpperCAmelCase,      __UpperCAmelCase	)		->					str:

					'''simple docstring'''

					snake_case_									=				LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
					print(F"Saving a {tokenizer_class.__name__} to {tokenizer_path}."	)
					snake_case_									=				tokenizer_class(__UpperCAmelCase	)
					tokenizer.save_pretrained(__UpperCAmelCase	)


def 		__magic_name__    (	)		->					Union[str, Any]:

					'''simple docstring'''

					snake_case_									=				argparse.ArgumentParser()
					parser.add_argument(
					    '''--input_dir''',      help='''Location of LLaMA weights, which contains tokenizer.model and model folders''',      )
					parser.add_argument(
					    '''--model_size''',      choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only'''],      )
					parser.add_argument(
					    '''--output_dir''',      help='''Location to write HF model and tokenizer''',      )
					parser.add_argument('''--safe_serialization''',      type=__UpperCAmelCase,      help='''Whether or not to save using `safetensors`.'''	)
					snake_case_									=				parser.parse_args()
					if args.model_size != "tokenizer_only":
										write_model(
										    model_path=args.output_dir,      input_base_path=os.path.join(args.input_dir,      args.model_size	),      model_size=args.model_size,      safe_serialization=args.safe_serialization,      )
					snake_case_									=				os.path.join(args.input_dir,      '''tokenizer.model'''	)
					write_tokenizer(args.output_dir,      __UpperCAmelCase	)


if __name__ == "__main__":
							main()







'''simple docstring'''

import re
from typing import Callable, List, Optional, Union

import tensorflow as tf


try:
							from tensorflow.keras.optimizers.legacy import Adam
except ImportError:
							from tensorflow.keras.optimizers import Adam






class 						a				(   tf.keras.optimizers.schedules.LearningRateSchedule     ):


				def __init__(		self    :					Optional[Any]       ,				lowercase_    :					float       ,				lowercase_    :					Callable       ,				lowercase_    :					int       ,				lowercase_    :					float = 1.0       ,				lowercase_    :					str = None       ,				):
									super().__init__()
									snake_case_									=				initial_learning_rate
									snake_case_									=				warmup_steps
									snake_case_									=				power
									snake_case_									=				decay_schedule_fn
									snake_case_									=				name


				def __call__(		self    :					Tuple       ,				lowercase_    :					str					):
									with tf.name_scope(self.name or '''WarmUp'''					) as name:
														# Implements polynomial warmup. i.e., if global_step < warmup_steps, the
														# learning rate will be `global_step/num_warmup_steps * init_lr`.
														snake_case_									=				tf.cast(lowercase_       ,				tf.floataa					)
														snake_case_									=				tf.cast(self.warmup_steps       ,				tf.floataa					)
														snake_case_									=				global_step_float / warmup_steps_float
														snake_case_									=				self.initial_learning_rate * tf.math.pow(lowercase_       ,				self.power					)
														return tf.cond(
														    global_step_float < warmup_steps_float       ,				lambda: warmup_learning_rate       ,				lambda: self.decay_schedule_fn(step - self.warmup_steps					)       ,				name=lowercase_       ,				)


				def   A_ (		self    :					Any					):
									return {
									    "initial_learning_rate": self.initial_learning_rate,
									    "decay_schedule_fn": self.decay_schedule_fn,
									    "warmup_steps": self.warmup_steps,
									    "power": self.power,
									    "name": self.name,
									}







def 		__magic_name__    (	__UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase = 0.0,      __UpperCAmelCase = 0.9,      __UpperCAmelCase = 0.9_9_9,      __UpperCAmelCase = 1e-8,      __UpperCAmelCase = None,      __UpperCAmelCase = None,      __UpperCAmelCase = 0.0,      __UpperCAmelCase = 1.0,      __UpperCAmelCase = None,      )		->					List[str]:

					'''simple docstring'''

					snake_case_									=				tf.keras.optimizers.schedules.PolynomialDecay(
					    initial_learning_rate=__UpperCAmelCase,      decay_steps=num_train_steps - num_warmup_steps,      end_learning_rate=init_lr * min_lr_ratio,      power=__UpperCAmelCase,      )
					if num_warmup_steps:
										snake_case_									=				WarmUp(
										    initial_learning_rate=__UpperCAmelCase,      decay_schedule_fn=__UpperCAmelCase,      warmup_steps=__UpperCAmelCase,      )
					if weight_decay_rate > 0.0:
										snake_case_									=				AdamWeightDecay(
										    learning_rate=__UpperCAmelCase,      weight_decay_rate=__UpperCAmelCase,      beta_a=__UpperCAmelCase,      beta_a=__UpperCAmelCase,      epsilon=__UpperCAmelCase,      clipnorm=__UpperCAmelCase,      global_clipnorm=__UpperCAmelCase,      exclude_from_weight_decay=['''LayerNorm''', '''layer_norm''', '''bias'''],      include_in_weight_decay=__UpperCAmelCase,      )
					else:
										snake_case_									=				tf.keras.optimizers.Adam(
										    learning_rate=__UpperCAmelCase,      beta_a=__UpperCAmelCase,      beta_a=__UpperCAmelCase,      epsilon=__UpperCAmelCase,      clipnorm=__UpperCAmelCase,      global_clipnorm=__UpperCAmelCase,      )
					# We return the optimizer and the LR scheduler in order to better track the
					# evolution of the LR independently of the optimizer.
					return optimizer, lr_schedule






class 						a				(   _lowerCamelCase     ):


				def __init__(		self    :					Dict       ,				lowercase_    :					Union[float, tf.keras.optimizers.schedules.LearningRateSchedule] = 0.001       ,				lowercase_    :					float = 0.9       ,				lowercase_    :					float = 0.999       ,				lowercase_    :					float = 1e-7       ,				lowercase_    :					bool = False       ,				lowercase_    :					float = 0.0       ,				lowercase_    :					Optional[List[str]] = None       ,				lowercase_    :					Optional[List[str]] = None       ,				lowercase_    :					str = "AdamWeightDecay"       ,				**lowercase_    :					Optional[int]       ,				):
									super().__init__(lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_       ,				**lowercase_					)
									snake_case_									=				weight_decay_rate
									snake_case_									=				include_in_weight_decay
									snake_case_									=				exclude_from_weight_decay


				@classmethod
				def   A_ (		cls    :					Dict       ,				lowercase_    :					Union[str, Any]					):
									snake_case_									=				{'''WarmUp''': WarmUp}
									return super(lowercase_       ,				cls					).from_config(lowercase_       ,				custom_objects=lowercase_					)


				def   A_ (		self    :					str       ,				lowercase_    :					Optional[int]       ,				lowercase_    :					List[str]       ,				lowercase_    :					Optional[int]					):
									super(lowercase_       ,				self					)._prepare_local(lowercase_       ,				lowercase_       ,				lowercase_					)
									snake_case_									=				tf.constant(
									    self.weight_decay_rate       ,				name='''adam_weight_decay_rate'''					)


				def   A_ (		self    :					str       ,				lowercase_    :					Union[str, Any]       ,				lowercase_    :					Dict       ,				lowercase_    :					Any					):
									snake_case_									=				self._do_use_weight_decay(var.name					)
									if do_decay:
														return var.assign_sub(
														    learning_rate * var * apply_state[(var.device, var.dtype.base_dtype)]['''weight_decay_rate''']       ,				use_locking=self._use_locking       ,				)
									return tf.no_op()


				def   A_ (		self    :					Union[str, Any]       ,				lowercase_    :					int       ,				lowercase_    :					str=None       ,				**lowercase_    :					List[str]					):
									snake_case_			,snake_case_									=				list(zip(*lowercase_					)					)
									return super(lowercase_       ,				self					).apply_gradients(zip(lowercase_       ,				lowercase_					)       ,				name=lowercase_       ,				**lowercase_					)


				def   A_ (		self    :					List[Any]       ,				lowercase_    :					str       ,				lowercase_    :					str       ,				lowercase_    :					Any					):
									if apply_state is None:
														return self._decayed_lr_t[var_dtype], {}

									snake_case_									=				apply_state or {}
									snake_case_									=				apply_state.get((var_device, var_dtype)					)
									if coefficients is None:
														snake_case_									=				self._fallback_apply_state(lowercase_       ,				lowercase_					)
														snake_case_									=				coefficients

									return coefficients["lr_t"], {"apply_state": apply_state}


				def   A_ (		self    :					Union[str, Any]       ,				lowercase_    :					int       ,				lowercase_    :					Union[str, Any]       ,				lowercase_    :					Optional[int]=None					):
									snake_case_			,snake_case_									=				self._get_lr(var.device       ,				var.dtype.base_dtype       ,				lowercase_					)
									snake_case_									=				self._decay_weights_op(lowercase_       ,				lowercase_       ,				lowercase_					)
									with tf.control_dependencies([decay]					):
														return super(lowercase_       ,				self					)._resource_apply_dense(lowercase_       ,				lowercase_       ,				**lowercase_					)


				def   A_ (		self    :					Optional[Any]       ,				lowercase_    :					Optional[Any]       ,				lowercase_    :					int       ,				lowercase_    :					str       ,				lowercase_    :					List[Any]=None					):
									snake_case_			,snake_case_									=				self._get_lr(var.device       ,				var.dtype.base_dtype       ,				lowercase_					)
									snake_case_									=				self._decay_weights_op(lowercase_       ,				lowercase_       ,				lowercase_					)
									with tf.control_dependencies([decay]					):
														return super(lowercase_       ,				self					)._resource_apply_sparse(lowercase_       ,				lowercase_       ,				lowercase_       ,				**lowercase_					)


				def   A_ (		self    :					Union[str, Any]					):
									snake_case_									=				super().get_config()
									config.update({'''weight_decay_rate''': self.weight_decay_rate}					)
									return config


				def   A_ (		self    :					Optional[int]       ,				lowercase_    :					int					):
									if self.weight_decay_rate == 0:
														return False

									if self._include_in_weight_decay:
														for r in self._include_in_weight_decay:
																			if re.search(lowercase_       ,				lowercase_					) is not None:
																								return True

									if self._exclude_from_weight_decay:
														for r in self._exclude_from_weight_decay:
																			if re.search(lowercase_       ,				lowercase_					) is not None:
																								return False
									return True







class 						a				(   _lowerCamelCase     ):


				def __init__(		self    :					List[Any]					):
									snake_case_									=				[]
									snake_case_									=				None


				@property
				def   A_ (		self    :					Union[str, Any]					):
									if self._accum_steps is None:
														snake_case_									=				tf.Variable(
														    tf.constant(0       ,				dtype=tf.intaa					)       ,				trainable=lowercase_       ,				synchronization=tf.VariableSynchronization.ON_READ       ,				aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA       ,				)

									return self._accum_steps.value()


				@property
				def   A_ (		self    :					Dict					):
									if not self._gradients:
														raise ValueError('''The accumulator should be called first to initialize the gradients'''					)
									return [gradient.value() if gradient is not None else gradient for gradient in self._gradients]


				def __call__(		self    :					Any       ,				lowercase_    :					int					):
									if not self._gradients:
														snake_case_									=				self.step  # Create the step variable.
														self._gradients.extend(
														    [
														        tf.Variable(
														            tf.zeros_like(lowercase_					)       ,				trainable=lowercase_       ,				synchronization=tf.VariableSynchronization.ON_READ       ,				aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA       ,				)
														        if gradient is not None
														        else gradient
														        for gradient in gradients
														    ]					)
									if len(lowercase_					) != len(self._gradients					):
														raise ValueError(F"Expected {len(self._gradients					)} gradients, but got {len(lowercase_					)}"					)

									for accum_gradient, gradient in zip(self._gradients       ,				lowercase_					):
														if accum_gradient is not None and gradient is not None:
																			accum_gradient.assign_add(lowercase_					)

									self._accum_steps.assign_add(1					)


				def   A_ (		self    :					Optional[int]					):
									if not self._gradients:
														return
									self._accum_steps.assign(0					)
									for gradient in self._gradients:
														if gradient is not None:
																			gradient.assign(tf.zeros_like(lowercase_					)					)

