

import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase					:			Tuple  				=			logging.get_logger(__name__)

_lowerCAmelCase					:			Optional[Any]  				=			"▁"

_lowerCAmelCase					:			Tuple  				=			{
    "vocab_file": "vocab.json",
    "spm_file": "sentencepiece.bpe.model",
    "tokenizer_config_file": "tokenizer_config.json",
}

_lowerCAmelCase					:			Any  				=			{
    "vocab_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json",
    },
    "spm_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model",
    },
    "tokenizer_config_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json",
    },
}

_lowerCAmelCase					:			Dict  				=			{
    "facebook/m2m100_418M": 1_024,
}

# fmt: off
_lowerCAmelCase					:			str  				=			{
    "m2m100": ["af", "am", "ar", "ast", "az", "ba", "be", "bg", "bn", "br", "bs", "ca", "ceb", "cs", "cy", "da", "de", "el", "en", "es", "et", "fa", "ff", "fi", "fr", "fy", "ga", "gd", "gl", "gu", "ha", "he", "hi", "hr", "ht", "hu", "hy", "id", "ig", "ilo", "is", "it", "ja", "jv", "ka", "kk", "km", "kn", "ko", "lb", "lg", "ln", "lo", "lt", "lv", "mg", "mk", "ml", "mn", "mr", "ms", "my", "ne", "nl", "no", "ns", "oc", "or", "pa", "pl", "ps", "pt", "ro", "ru", "sd", "si", "sk", "sl", "so", "sq", "sr", "ss", "su", "sv", "sw", "ta", "th", "tl", "tn", "tr", "uk", "ur", "uz", "vi", "wo", "xh", "yi", "yo", "zh", "zu"],
    "wmt21": ["en", "ha", "is", "ja", "cs", "ru", "zh", "de"]
}






class 	__magic_name__    (			A__      ):
			SCREAMING_SNAKE_CASE							   =						VOCAB_FILES_NAMES
			SCREAMING_SNAKE_CASE							   =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			SCREAMING_SNAKE_CASE							   =						PRETRAINED_VOCAB_FILES_MAP
			SCREAMING_SNAKE_CASE							   =						['input_ids', 'attention_mask']

			SCREAMING_SNAKE_CASE							   =						[]
			SCREAMING_SNAKE_CASE							   =						[]





			def __init__(		self					,  __snake_case					,  __snake_case					,  __snake_case=None					,  __snake_case=None					,  __snake_case="<s>"					,  __snake_case="</s>"					,  __snake_case="</s>"					,  __snake_case="<pad>"					,  __snake_case="<unk>"					,  __snake_case="m2m100"					,  __snake_case = None					,  __snake_case=8					,  **__snake_case					,  )   -> None:




						'''simple docstring'''
						__a        ={} if sp_model_kwargs is None else sp_model_kwargs

						__a        =language_codes
						__a        =FAIRSEQ_LANGUAGE_CODES[language_codes]
						__a        ={lang_code: f'__{lang_code}__' for lang_code in fairseq_language_code}

						__a        =kwargs.get('additional_special_tokens'					,  []					)
						kwargs["additional_special_tokens"] += [
						    self.get_lang_token(__snake_case					)
						    for lang_code in fairseq_language_code
						    if self.get_lang_token(__snake_case					) not in kwargs["additional_special_tokens"]
						]

						super().__init__(
						    src_lang=__snake_case					,  tgt_lang=__snake_case					,  bos_token=__snake_case					,  eos_token=__snake_case					,  sep_token=__snake_case					,  unk_token=__snake_case					,  pad_token=__snake_case					,  language_codes=__snake_case					,  sp_model_kwargs=self.sp_model_kwargs					,  num_madeup_words=__snake_case					,  **__snake_case					,  )

						__a        =vocab_file
						__a        =load_json(__snake_case					)
						__a        ={v: k for k, v in self.encoder.items()}
						__a        =spm_file
						__a        =load_spm(__snake_case					,  self.sp_model_kwargs					)

						__a        =len(self.encoder					)

						__a        ={
						    self.get_lang_token(__snake_case					): self.encoder_size + i for i, lang_code in enumerate(__snake_case					)
						}
						__a        ={lang_code: self.encoder_size + i for i, lang_code in enumerate(__snake_case					)}
						__a        ={v: k for k, v in self.lang_token_to_id.items()}

						__a        =src_lang if src_lang is not None else '''en'''
						__a        =tgt_lang
						__a        =self.get_lang_id(self._src_lang					)
						self.set_src_lang_special_tokens(self._src_lang					)

						__a        =num_madeup_words





			@property
			def 		__magic_name__    (		self					)   -> int:




						'''simple docstring'''
						return len(self.encoder					) + len(self.lang_token_to_id					)





			@property
			def 		__magic_name__    (		self					)   -> str:




						'''simple docstring'''
						return self._src_lang





			@src_lang.setter
			def 		__magic_name__    (		self					,  __snake_case					)   -> None:




						'''simple docstring'''
						__a        =new_src_lang
						self.set_src_lang_special_tokens(self._src_lang					)





			def 		__magic_name__    (		self					,  __snake_case					)   -> List[str]:




						'''simple docstring'''
						return self.sp_model.encode(__snake_case					,  out_type=__snake_case					)





			def 		__magic_name__    (		self					,  __snake_case					)   -> Tuple:




						'''simple docstring'''
						if token in self.lang_token_to_id:
									return self.lang_token_to_id[token]
						return self.encoder.get(__snake_case					,  self.encoder[self.unk_token]					)





			def 		__magic_name__    (		self					,  __snake_case					)   -> str:




						'''simple docstring'''
						if index in self.id_to_lang_token:
									return self.id_to_lang_token[index]
						return self.decoder.get(__snake_case					,  self.unk_token					)





			def 		__magic_name__    (		self					,  __snake_case					)   -> Tuple:




						'''simple docstring'''
						__a        =[]
						__a        =''''''
						for token in tokens:
									# make sure that special tokens are not decoded using sentencepiece model
									if token in self.all_special_tokens:
												out_string += self.sp_model.decode(__snake_case					) + token
												__a        =[]
									else:
												current_sub_tokens.append(__snake_case					)
						out_string += self.sp_model.decode(__snake_case					)
						return out_string.strip()





			def 		__magic_name__    (		self					,  __snake_case					,  __snake_case = None					,  __snake_case = False					)   -> List[int]:




						'''simple docstring'''

						if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=__snake_case					,  token_ids_a=__snake_case					,  already_has_special_tokens=__snake_case					)

						__a        =[1] * len(self.prefix_tokens					)
						__a        =[1] * len(self.suffix_tokens					)
						if token_ids_a is None:
									return prefix_ones + ([0] * len(__snake_case					)) + suffix_ones
						return prefix_ones + ([0] * len(__snake_case					)) + ([0] * len(__snake_case					)) + suffix_ones





			def 		__magic_name__    (		self					,  __snake_case					,  __snake_case = None					)   -> List[int]:




						'''simple docstring'''
						if token_ids_a is None:
									return self.prefix_tokens + token_ids_a + self.suffix_tokens
						# We don't expect to process pairs, but leave the pair logic for API consistency
						return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens





			def 		__magic_name__    (		self					)   -> Dict:




						'''simple docstring'''
						__a        ={self.convert_ids_to_tokens(__snake_case					): i for i in range(self.vocab_size					)}
						vocab.update(self.added_tokens_encoder					)
						return vocab





			def __getstate__(		self					)   -> Dict:




						'''simple docstring'''
						__a        =self.__dict__.copy()
						__a        =None
						return state





			def __setstate__(		self					,  __snake_case					)   -> None:




						'''simple docstring'''
						__a        =d

						# for backward compatibility
						if not hasattr(self					,  'sp_model_kwargs'					):
									__a        ={}

						__a        =load_spm(self.spm_file					,  self.sp_model_kwargs					)





			def 		__magic_name__    (		self					,  __snake_case					,  __snake_case = None					)   -> Tuple[str]:




						'''simple docstring'''
						__a        =Path(__snake_case					)
						if not save_dir.is_dir():
									raise OSError(f'{save_directory} should be a directory'					)
						__a        =save_dir / (
						    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
						)
						__a        =save_dir / (
						    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
						)

						save_json(self.encoder					,  __snake_case					)

						if os.path.abspath(self.spm_file					) != os.path.abspath(__snake_case					) and os.path.isfile(self.spm_file					):
									copyfile(self.spm_file					,  __snake_case					)
						elif not os.path.isfile(self.spm_file					):
									with open(__snake_case					,  'wb'					) as fi:
												__a        =self.sp_model.serialized_model_proto()
												fi.write(__snake_case					)

						return (str(__snake_case					), str(__snake_case					))





			def 		__magic_name__    (		self					,  __snake_case					,  __snake_case = "en"					,  __snake_case = None					,  __snake_case = "ro"					,  **__snake_case					,  )   -> BatchEncoding:




						'''simple docstring'''
						__a        =src_lang
						__a        =tgt_lang
						self.set_src_lang_special_tokens(self.src_lang					)
						return super().prepare_seqaseq_batch(__snake_case					,  __snake_case					,  **__snake_case					)





			def 		__magic_name__    (		self					,  __snake_case					,  __snake_case					,  __snake_case					,  **__snake_case					)   -> Optional[int]:




						'''simple docstring'''
						if src_lang is None or tgt_lang is None:
									raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'					)
						__a        =src_lang
						__a        =self(__snake_case					,  add_special_tokens=__snake_case					,  **__snake_case					)
						__a        =self.get_lang_id(__snake_case					)
						__a        =tgt_lang_id
						return inputs





			def 		__magic_name__    (		self					)   -> int:




						'''simple docstring'''
						self.set_src_lang_special_tokens(self.src_lang					)





			def 		__magic_name__    (		self					)   -> int:




						'''simple docstring'''
						self.set_tgt_lang_special_tokens(self.tgt_lang					)





			def 		__magic_name__    (		self					,  __snake_case					)   -> None:




						'''simple docstring'''
						__a        =self.get_lang_token(__snake_case					)
						__a        =self.lang_token_to_id[lang_token]
						__a        =[self.cur_lang_id]
						__a        =[self.eos_token_id]





			def 		__magic_name__    (		self					,  __snake_case					)   -> None:




						'''simple docstring'''
						__a        =self.get_lang_token(__snake_case					)
						__a        =self.lang_token_to_id[lang_token]
						__a        =[self.cur_lang_id]
						__a        =[self.eos_token_id]





			def 		__magic_name__    (		self					,  __snake_case					)   -> str:




						'''simple docstring'''
						return self.lang_code_to_token[lang]





			def 		__magic_name__    (		self					,  __snake_case					)   -> int:




						'''simple docstring'''
						__a        =self.get_lang_token(__snake_case					)
						return self.lang_token_to_id[lang_token]






def        UpperCamelCase_(							_snake_case   :  str       ,     _snake_case   :  Dict[str, Any]	):




			"""simple docstring"""






			__a        =sentencepiece.SentencePieceProcessor(**_lowerCAmelCase	)
			spm.Load(str(_lowerCAmelCase	)	)
			return spm






def        UpperCamelCase_(							_snake_case   :  str	):




			"""simple docstring"""






			with open(_lowerCAmelCase       ,     'r'	) as f:
						return json.load(_lowerCAmelCase	)






def        UpperCamelCase_(							_snake_case   :  List[str]       ,     _snake_case   :  str	):




			"""simple docstring"""






			with open(_lowerCAmelCase       ,     'w'	) as f:
						json.dump(_lowerCAmelCase       ,     _lowerCAmelCase       ,     indent=2	)











'''simple docstring'''






import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
						from transformers import LlamaTokenizerFast
except ImportError as e:
						warnings.warn(e)
						warnings.warn(
						    "The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"
						)
						UpperCamelCase__:   str								=			None
UpperCamelCase__:   int								=			{
    "7B": 11008,
    "13B": 13824,
    "30B": 17920,
    "65B": 22016,
    "70B": 28672,
}
UpperCamelCase__:   List[Any]								=			{
    "7B": 1,
    "7Bf": 1,
    "13B": 2,
    "13Bf": 2,
    "30B": 4,
    "65B": 8,
    "70B": 8,
    "70Bf": 8,
}


def  snake_case_     (					_lowerCAmelCase  :						Tuple   ,      _lowerCAmelCase  :						Tuple=1   ,      _lowerCAmelCase  :						List[Any]=256  )      ->   Optional[Any]:
							return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3  )  ) + multiple_of - 1) // multiple_of)


def  snake_case_     (					_lowerCAmelCase  :						List[str]  )      ->   str:
							with open(_lowerCAmelCase   ,      '''r'''  ) as f:
														return json.load(_lowerCAmelCase  )


def  snake_case_     (					_lowerCAmelCase  :						Union[str, Any]   ,      _lowerCAmelCase  :						Any  )      ->   Optional[Any]:
							with open(_lowerCAmelCase   ,      '''w'''  ) as f:
														json.dump(_lowerCAmelCase   ,      _lowerCAmelCase  )


def  snake_case_     (					_lowerCAmelCase  :						Dict   ,      _lowerCAmelCase  :						Tuple   ,      _lowerCAmelCase  :						Any   ,      _lowerCAmelCase  :						Any=True  )      ->   List[Any]:
							os.makedirs(_lowerCAmelCase   ,      exist_ok=_lowerCAmelCase  )
							UpperCAmelCase :       int 							=		os.path.join(_lowerCAmelCase   ,      '''tmp'''  )
							os.makedirs(_lowerCAmelCase   ,      exist_ok=_lowerCAmelCase  )

							UpperCAmelCase :       List[str] 							=		read_json(os.path.join(_lowerCAmelCase   ,      '''params.json'''  )  )
							UpperCAmelCase :       str 							=		NUM_SHARDS[model_size]
							UpperCAmelCase :       Any 							=		params['''n_layers''']
							UpperCAmelCase :       str 							=		params['''n_heads''']
							UpperCAmelCase :       Any 							=		n_heads // num_shards
							UpperCAmelCase :       List[str] 							=		params['''dim''']
							UpperCAmelCase :       Optional[Any] 							=		dim // n_heads
							UpperCAmelCase :       str 							=		1_0_0_0_0.0
							UpperCAmelCase :       Optional[int] 							=		1.0 / (base ** (torch.arange(0   ,      _lowerCAmelCase   ,      2  ).float() / dims_per_head))

							if "n_kv_heads" in params:
														UpperCAmelCase :       Tuple 							=		params['''n_kv_heads''']  # for GQA / MQA
														UpperCAmelCase :       Optional[int] 							=		n_heads_per_shard // num_key_value_heads
														UpperCAmelCase :       Optional[Any] 							=		dim // num_key_value_heads
							else:  # compatibility with other checkpoints
														UpperCAmelCase :       List[str] 							=		n_heads
														UpperCAmelCase :       Optional[int] 							=		n_heads_per_shard
														UpperCAmelCase :       List[str] 							=		dim

							# permute for sliced rotary
							def permute(_lowerCAmelCase  :						Tuple   ,      _lowerCAmelCase  :						Optional[Any]=n_heads   ,      _lowerCAmelCase  :						int=dim   ,      _lowerCAmelCase  :						Dict=dim  ):
														return w.view(_lowerCAmelCase   ,      dima // n_heads // 2   ,      2   ,      _lowerCAmelCase  ).transpose(1   ,      2  ).reshape(_lowerCAmelCase   ,      _lowerCAmelCase  )

							print(f"""Fetching all parameters from the checkpoint at {input_base_path}."""  )
							# Load weights
							if model_size == "7B":
														# Not sharded
														# (The sharded implementation would also work, but this is simpler.)
														UpperCAmelCase :       int 							=		torch.load(os.path.join(_lowerCAmelCase   ,      '''consolidated.00.pth'''  )   ,      map_location='''cpu'''  )
							else:
														# Sharded
														UpperCAmelCase :       Optional[Any] 							=		[
														    torch.load(os.path.join(_lowerCAmelCase   ,      f"""consolidated.{i:02d}.pth"""  )   ,      map_location='''cpu'''  )
														    for i in range(_lowerCAmelCase  )
														]
							UpperCAmelCase :       Any 							=		0
							UpperCAmelCase :       str 							=		{'''weight_map''': {}}
							for layer_i in range(_lowerCAmelCase  ):
														UpperCAmelCase :       Optional[Any] 							=		f"""pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"""
														if model_size == "7B":
																					# Unsharded
																					UpperCAmelCase :       Optional[int] 							=		{
																					    f"""model.layers.{layer_i}.self_attn.q_proj.weight""": permute(
																					        loaded[f"""layers.{layer_i}.attention.wq.weight"""]  ),
																					    f"""model.layers.{layer_i}.self_attn.k_proj.weight""": permute(
																					        loaded[f"""layers.{layer_i}.attention.wk.weight"""]  ),
																					    f"""model.layers.{layer_i}.self_attn.v_proj.weight""": loaded[f"""layers.{layer_i}.attention.wv.weight"""],
																					    f"""model.layers.{layer_i}.self_attn.o_proj.weight""": loaded[f"""layers.{layer_i}.attention.wo.weight"""],
																					    f"""model.layers.{layer_i}.mlp.gate_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w1.weight"""],
																					    f"""model.layers.{layer_i}.mlp.down_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w2.weight"""],
																					    f"""model.layers.{layer_i}.mlp.up_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w3.weight"""],
																					    f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[f"""layers.{layer_i}.attention_norm.weight"""],
																					    f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[f"""layers.{layer_i}.ffn_norm.weight"""],
																					}
														else:
																					# Sharded
																					# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
																					# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
																					# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

																					UpperCAmelCase :       List[str] 							=		{
																					    f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[0][
																					        f"""layers.{layer_i}.attention_norm.weight"""
																					    ].clone(),
																					    f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[0][
																					        f"""layers.{layer_i}.ffn_norm.weight"""
																					    ].clone(),
																					}
																					UpperCAmelCase :       Union[str, Any] 							=		permute(
																					    torch.cat(
																					        [
																					            loaded[i][f"""layers.{layer_i}.attention.wq.weight"""].view(_lowerCAmelCase   ,      _lowerCAmelCase   ,      _lowerCAmelCase  )
																					            for i in range(_lowerCAmelCase  )
																					        ]   ,      dim=0   ,      ).reshape(_lowerCAmelCase   ,      _lowerCAmelCase  )  )
																					UpperCAmelCase :       Optional[Any] 							=		permute(
																					    torch.cat(
																					        [
																					            loaded[i][f"""layers.{layer_i}.attention.wk.weight"""].view(
																					                _lowerCAmelCase   ,      _lowerCAmelCase   ,      _lowerCAmelCase  )
																					            for i in range(_lowerCAmelCase  )
																					        ]   ,      dim=0   ,      ).reshape(_lowerCAmelCase   ,      _lowerCAmelCase  )   ,      _lowerCAmelCase   ,      _lowerCAmelCase   ,      _lowerCAmelCase   ,      )
																					UpperCAmelCase :       str 							=		torch.cat(
																					    [
																					        loaded[i][f"""layers.{layer_i}.attention.wv.weight"""].view(
																					            _lowerCAmelCase   ,      _lowerCAmelCase   ,      _lowerCAmelCase  )
																					        for i in range(_lowerCAmelCase  )
																					    ]   ,      dim=0   ,      ).reshape(_lowerCAmelCase   ,      _lowerCAmelCase  )

																					UpperCAmelCase :       Optional[int] 							=		torch.cat(
																					    [loaded[i][f"""layers.{layer_i}.attention.wo.weight"""] for i in range(_lowerCAmelCase  )]   ,      dim=1  )
																					UpperCAmelCase :       Any 							=		torch.cat(
																					    [loaded[i][f"""layers.{layer_i}.feed_forward.w1.weight"""] for i in range(_lowerCAmelCase  )]   ,      dim=0  )
																					UpperCAmelCase :       str 							=		torch.cat(
																					    [loaded[i][f"""layers.{layer_i}.feed_forward.w2.weight"""] for i in range(_lowerCAmelCase  )]   ,      dim=1  )
																					UpperCAmelCase :       Tuple 							=		torch.cat(
																					    [loaded[i][f"""layers.{layer_i}.feed_forward.w3.weight"""] for i in range(_lowerCAmelCase  )]   ,      dim=0  )

														UpperCAmelCase :       Any 							=		inv_freq
														for k, v in state_dict.items():
																					UpperCAmelCase :       List[Any] 							=		filename
																					param_count += v.numel()
														torch.save(_lowerCAmelCase   ,      os.path.join(_lowerCAmelCase   ,      _lowerCAmelCase  )  )

							UpperCAmelCase :       Optional[int] 							=		f"""pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"""
							if model_size == "7B":
														# Unsharded
														UpperCAmelCase :       str 							=		{
														    '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
														    '''model.norm.weight''': loaded['''norm.weight'''],
														    '''lm_head.weight''': loaded['''output.weight'''],
														}
							else:
														UpperCAmelCase :       Any 							=		{
														    '''model.norm.weight''': loaded[0]['''norm.weight'''],
														    '''model.embed_tokens.weight''': torch.cat(
														        [loaded[i]['''tok_embeddings.weight'''] for i in range(_lowerCAmelCase  )]   ,      dim=1  ),
														    '''lm_head.weight''': torch.cat([loaded[i]['''output.weight'''] for i in range(_lowerCAmelCase  )]   ,      dim=0  ),
														}

							for k, v in state_dict.items():
														UpperCAmelCase :       Optional[int] 							=		filename
														param_count += v.numel()
							torch.save(_lowerCAmelCase   ,      os.path.join(_lowerCAmelCase   ,      _lowerCAmelCase  )  )

							# Write configs
							UpperCAmelCase :       Union[str, Any] 							=		{'''total_size''': param_count * 2}
							write_json(_lowerCAmelCase   ,      os.path.join(_lowerCAmelCase   ,      '''pytorch_model.bin.index.json'''  )  )
							UpperCAmelCase :       int 							=		params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
							UpperCAmelCase :       Tuple 							=		params['''multiple_of'''] if '''multiple_of''' in params else 256
							UpperCAmelCase :       Any 							=		LlamaConfig(
							    hidden_size=_lowerCAmelCase   ,      intermediate_size=compute_intermediate_size(_lowerCAmelCase   ,      _lowerCAmelCase   ,      _lowerCAmelCase  )   ,      num_attention_heads=params['''n_heads''']   ,      num_hidden_layers=params['''n_layers''']   ,      rms_norm_eps=params['''norm_eps''']   ,      num_key_value_heads=_lowerCAmelCase   ,      )
							config.save_pretrained(_lowerCAmelCase  )

							# Make space so we can load the model properly now.
							del state_dict
							del loaded
							gc.collect()

							print('''Loading the checkpoint in a Llama model.'''  )
							UpperCAmelCase :       Optional[int] 							=		LlamaForCausalLM.from_pretrained(_lowerCAmelCase   ,      torch_dtype=torch.floataa   ,      low_cpu_mem_usage=_lowerCAmelCase  )
							# Avoid saving this as part of the config.
							del model.config._name_or_path

							print('''Saving in the Transformers format.'''  )
							model.save_pretrained(_lowerCAmelCase   ,      safe_serialization=_lowerCAmelCase  )
							shutil.rmtree(_lowerCAmelCase  )


def  snake_case_     (					_lowerCAmelCase  :						int   ,      _lowerCAmelCase  :						Union[str, Any]  )      ->   List[str]:
							# Initialize the tokenizer based on the `spm` model
							UpperCAmelCase :       Dict 							=		LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
							print(f"""Saving a {tokenizer_class.__name__} to {tokenizer_path}."""  )
							UpperCAmelCase :       List[Any] 							=		tokenizer_class(_lowerCAmelCase  )
							tokenizer.save_pretrained(_lowerCAmelCase  )


def  snake_case_     (					)      ->   List[Any]:
							UpperCAmelCase :       int 							=		argparse.ArgumentParser()
							parser.add_argument(
							    '''--input_dir'''   ,      help='''Location of LLaMA weights, which contains tokenizer.model and model folders'''   ,      )
							parser.add_argument(
							    '''--model_size'''   ,      choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only''']   ,      )
							parser.add_argument(
							    '''--output_dir'''   ,      help='''Location to write HF model and tokenizer'''   ,      )
							parser.add_argument('''--safe_serialization'''   ,      type=_lowerCAmelCase   ,      help='''Whether or not to save using `safetensors`.'''  )
							UpperCAmelCase :       List[Any] 							=		parser.parse_args()
							if args.model_size != "tokenizer_only":
														write_model(
														    model_path=args.output_dir   ,      input_base_path=os.path.join(args.input_dir   ,      args.model_size  )   ,      model_size=args.model_size   ,      safe_serialization=args.safe_serialization   ,      )
							UpperCAmelCase :       Optional[int] 							=		os.path.join(args.input_dir   ,      '''tokenizer.model'''  )
							write_tokenizer(args.output_dir   ,      _lowerCAmelCase  )


if __name__ == "__main__":
						main()





