



'''simple docstring'''






from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...utils import logging


if TYPE_CHECKING:
						from ...feature_extraction_utils import FeatureExtractionMixin
						from ...tokenization_utils_base import PreTrainedTokenizerBase
						from ...utils import TensorType

UpperCamelCase__:   Any								=			logging.get_logger(__name__)

UpperCamelCase__:   Union[str, Any]								=			{
    "openai/whisper-base": "https://huggingface.co/openai/whisper-base/resolve/main/config.json",
}

# fmt: off
UpperCamelCase__:   Optional[Any]								=			[
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 357, 366, 438, 532, 685,
    705, 796, 930, 1058, 1220, 1267, 1279, 1303, 1343, 1377,
    1391, 1635, 1782, 1875, 2162, 2361, 2488, 3467, 4008, 4211,
    4600, 4808, 5299, 5855, 6329, 7203, 9609, 9959, 10563, 10786,
    11420, 11709, 11907, 13163, 13697, 13700, 14808, 15306, 16410, 16791,
    17992, 19203, 19510, 20724, 22305, 22935, 27007, 30109, 30420, 33409,
    34949, 40283, 40493, 40549, 47282, 49146, 50257, 50359, 50360, 50361
]
UpperCamelCase__:   List[str]								=			[
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 359, 503, 522, 542, 873,
    893, 902, 918, 922, 931, 1350, 1853, 1982, 2460, 2627,
    3246, 3253, 3268, 3536, 3846, 3961, 4183, 4667, 6585, 6647,
    7273, 9061, 9383, 10428, 10929, 11938, 12033, 12331, 12562, 13793,
    14157, 14635, 15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675,
    22520, 26130, 26161, 26435, 28279, 29464, 31650, 32302, 32470, 36865,
    42863, 47425, 49870, 50254, 50258, 50360, 50361, 50362
]

class 				SCREAMING_SNAKE_CASE(							A__			):







	"""simple docstring"""






	lowerCamelCase__			 =       """whisper"""
	lowerCamelCase__			 =       ["""past_key_values"""]
	lowerCamelCase__			 =       {"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}







	def __init__(						self			:     Dict		,			__snake_case			:     Tuple=51865		,			__snake_case			:     Union[str, Any]=80		,			__snake_case			:     str=6		,			__snake_case			:     int=4		,			__snake_case			:     Optional[Any]=6		,			__snake_case			:     Tuple=4		,			__snake_case			:     Optional[Any]=1536		,			__snake_case			:     Tuple=1536		,			__snake_case			:     Union[str, Any]=0.0		,			__snake_case			:     List[str]=0.0		,			__snake_case			:     Optional[int]=50257		,			__snake_case			:     Dict=True		,			__snake_case			:     int=True		,			__snake_case			:     Optional[int]="gelu"		,			__snake_case			:     Tuple=256		,			__snake_case			:     Any=0.0		,			__snake_case			:     List[Any]=0.0		,			__snake_case			:     str=0.0		,			__snake_case			:     str=0.02		,			__snake_case			:     List[str]=False		,			__snake_case			:     Any=1500		,			__snake_case			:     List[Any]=448		,			__snake_case			:     Any=50256		,			__snake_case			:     List[Any]=50256		,			__snake_case			:     Tuple=50256		,			__snake_case			:     Optional[Any]=None		,			__snake_case			:     str=[220, 50256]		,			__snake_case			:     Tuple=False		,			__snake_case			:     Dict=256		,			__snake_case			:     Tuple=False		,			__snake_case			:     Tuple=0.05		,			__snake_case			:     int=10		,			__snake_case			:     str=2		,			__snake_case			:     Optional[Any]=0.0		,			__snake_case			:     str=10		,			__snake_case			:     Optional[int]=0		,			__snake_case			:     Optional[int]=7		,			**__snake_case			:     Optional[int]		,			)			->		List[str]:
								UpperCAmelCase :       List[str] 							=		vocab_size
								UpperCAmelCase :       int 							=		num_mel_bins
								UpperCAmelCase :       Optional[int] 							=		d_model
								UpperCAmelCase :       str 							=		encoder_layers
								UpperCAmelCase :       Tuple 							=		encoder_attention_heads
								UpperCAmelCase :       Optional[Any] 							=		decoder_layers
								UpperCAmelCase :       List[str] 							=		decoder_attention_heads
								UpperCAmelCase :       int 							=		decoder_ffn_dim
								UpperCAmelCase :       List[str] 							=		encoder_ffn_dim
								UpperCAmelCase :       List[str] 							=		dropout
								UpperCAmelCase :       List[str] 							=		attention_dropout
								UpperCAmelCase :       Optional[int] 							=		activation_dropout
								UpperCAmelCase :       Optional[int] 							=		activation_function
								UpperCAmelCase :       str 							=		init_std
								UpperCAmelCase :       Union[str, Any] 							=		encoder_layerdrop
								UpperCAmelCase :       Dict 							=		decoder_layerdrop
								UpperCAmelCase :       Union[str, Any] 							=		use_cache
								UpperCAmelCase :       List[Any] 							=		encoder_layers
								UpperCAmelCase :       List[Any] 							=		scale_embedding  # scale factor will be sqrt(d_model) if True
								UpperCAmelCase :       str 							=		max_source_positions
								UpperCAmelCase :       List[Any] 							=		max_target_positions

								# Audio Classification-specific parameters. Feel free to ignore for other classes.
								UpperCAmelCase :       str 							=		classifier_proj_size
								UpperCAmelCase :       Optional[int] 							=		use_weighted_layer_sum

								# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
								UpperCAmelCase :       str 							=		apply_spec_augment
								UpperCAmelCase :       List[Any] 							=		mask_time_prob
								UpperCAmelCase :       str 							=		mask_time_length
								UpperCAmelCase :       Union[str, Any] 							=		mask_time_min_masks
								UpperCAmelCase :       str 							=		mask_feature_prob
								UpperCAmelCase :       List[Any] 							=		mask_feature_length
								UpperCAmelCase :       List[str] 							=		mask_feature_min_masks

								UpperCAmelCase :       Dict 							=		median_filter_width

								super().__init__(
								    pad_token_id=__snake_case		,			bos_token_id=__snake_case		,			eos_token_id=__snake_case		,			is_encoder_decoder=__snake_case		,			decoder_start_token_id=__snake_case		,			suppress_tokens=__snake_case		,			begin_suppress_tokens=__snake_case		,			**__snake_case		,			)

class 				SCREAMING_SNAKE_CASE(							A__			):







	"""simple docstring"""






	@property
	def      A	(						self			:     Optional[int]		)			->		Mapping[str, Mapping[int, str]]:
								UpperCAmelCase :       Dict 							=		OrderedDict(
								    [
								        ('''input_features''', {0: '''batch''', 1: '''feature_size''', 2: '''encoder_sequence'''}),
								    ]		)
								if self.use_past:
															UpperCAmelCase :       Any 							=		{0: '''batch'''}
								else:
															UpperCAmelCase :       List[Any] 							=		{0: '''batch''', 1: '''decoder_sequence'''}

								if self.use_past:
															self.fill_with_past_key_values_(__snake_case		,			direction='''inputs'''		)

								return common_inputs







	def      A	(						self			:     List[str]		,			__snake_case			:     Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"]		,			__snake_case			:     int = -1		,			__snake_case			:     int = -1		,			__snake_case			:     bool = False		,			__snake_case			:     Optional["TensorType"] = None		,			__snake_case			:     int = 22050		,			__snake_case			:     float = 5.0		,			__snake_case			:     int = 220		,			)			->		Mapping[str, Any]:
								UpperCAmelCase :       int 							=		OrderedDict()
								UpperCAmelCase :       List[str] 							=		OnnxConfig.generate_dummy_inputs(
								    self		,			preprocessor=preprocessor.feature_extractor		,			batch_size=__snake_case		,			framework=__snake_case		,			sampling_rate=__snake_case		,			time_duration=__snake_case		,			frequency=__snake_case		,			)
								UpperCAmelCase :       int 							=		encoder_inputs['''input_features'''].shape[2]
								UpperCAmelCase :       Optional[int] 							=		encoder_sequence_length // 2 if self.use_past else seq_length

								UpperCAmelCase :       Optional[Any] 							=		super().generate_dummy_inputs(
								    preprocessor.tokenizer		,			__snake_case		,			__snake_case		,			__snake_case		,			__snake_case		)

								UpperCAmelCase :       Optional[int] 							=		encoder_inputs.pop('''input_features'''		)
								UpperCAmelCase :       int 							=		decoder_inputs.pop('''decoder_input_ids'''		)

								if "past_key_values" in decoder_inputs:
															UpperCAmelCase :       Union[str, Any] 							=		decoder_inputs.pop('''past_key_values'''		)

								return dummy_inputs







	@property
	def      A	(						self			:     List[Any]		)			->		float:
								return 1E-3










'''simple docstring'''






import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
						from transformers import LlamaTokenizerFast
except ImportError as e:
						warnings.warn(e)
						warnings.warn(
						    "The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"
						)
						UpperCamelCase__:   str								=			None
UpperCamelCase__:   int								=			{
    "7B": 11008,
    "13B": 13824,
    "30B": 17920,
    "65B": 22016,
    "70B": 28672,
}
UpperCamelCase__:   List[Any]								=			{
    "7B": 1,
    "7Bf": 1,
    "13B": 2,
    "13Bf": 2,
    "30B": 4,
    "65B": 8,
    "70B": 8,
    "70Bf": 8,
}


def  snake_case_     (					_lowerCAmelCase  :						Tuple   ,      _lowerCAmelCase  :						Tuple=1   ,      _lowerCAmelCase  :						List[Any]=256  )      ->   Optional[Any]:
							return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3  )  ) + multiple_of - 1) // multiple_of)


def  snake_case_     (					_lowerCAmelCase  :						List[str]  )      ->   str:
							with open(_lowerCAmelCase   ,      '''r'''  ) as f:
														return json.load(_lowerCAmelCase  )


def  snake_case_     (					_lowerCAmelCase  :						Union[str, Any]   ,      _lowerCAmelCase  :						Any  )      ->   Optional[Any]:
							with open(_lowerCAmelCase   ,      '''w'''  ) as f:
														json.dump(_lowerCAmelCase   ,      _lowerCAmelCase  )


def  snake_case_     (					_lowerCAmelCase  :						Dict   ,      _lowerCAmelCase  :						Tuple   ,      _lowerCAmelCase  :						Any   ,      _lowerCAmelCase  :						Any=True  )      ->   List[Any]:
							os.makedirs(_lowerCAmelCase   ,      exist_ok=_lowerCAmelCase  )
							UpperCAmelCase :       int 							=		os.path.join(_lowerCAmelCase   ,      '''tmp'''  )
							os.makedirs(_lowerCAmelCase   ,      exist_ok=_lowerCAmelCase  )

							UpperCAmelCase :       List[str] 							=		read_json(os.path.join(_lowerCAmelCase   ,      '''params.json'''  )  )
							UpperCAmelCase :       str 							=		NUM_SHARDS[model_size]
							UpperCAmelCase :       Any 							=		params['''n_layers''']
							UpperCAmelCase :       str 							=		params['''n_heads''']
							UpperCAmelCase :       Any 							=		n_heads // num_shards
							UpperCAmelCase :       List[str] 							=		params['''dim''']
							UpperCAmelCase :       Optional[Any] 							=		dim // n_heads
							UpperCAmelCase :       str 							=		1_0_0_0_0.0
							UpperCAmelCase :       Optional[int] 							=		1.0 / (base ** (torch.arange(0   ,      _lowerCAmelCase   ,      2  ).float() / dims_per_head))

							if "n_kv_heads" in params:
														UpperCAmelCase :       Tuple 							=		params['''n_kv_heads''']  # for GQA / MQA
														UpperCAmelCase :       Optional[int] 							=		n_heads_per_shard // num_key_value_heads
														UpperCAmelCase :       Optional[Any] 							=		dim // num_key_value_heads
							else:  # compatibility with other checkpoints
														UpperCAmelCase :       List[str] 							=		n_heads
														UpperCAmelCase :       Optional[int] 							=		n_heads_per_shard
														UpperCAmelCase :       List[str] 							=		dim

							# permute for sliced rotary
							def permute(_lowerCAmelCase  :						Tuple   ,      _lowerCAmelCase  :						Optional[Any]=n_heads   ,      _lowerCAmelCase  :						int=dim   ,      _lowerCAmelCase  :						Dict=dim  ):
														return w.view(_lowerCAmelCase   ,      dima // n_heads // 2   ,      2   ,      _lowerCAmelCase  ).transpose(1   ,      2  ).reshape(_lowerCAmelCase   ,      _lowerCAmelCase  )

							print(f"""Fetching all parameters from the checkpoint at {input_base_path}."""  )
							# Load weights
							if model_size == "7B":
														# Not sharded
														# (The sharded implementation would also work, but this is simpler.)
														UpperCAmelCase :       int 							=		torch.load(os.path.join(_lowerCAmelCase   ,      '''consolidated.00.pth'''  )   ,      map_location='''cpu'''  )
							else:
														# Sharded
														UpperCAmelCase :       Optional[Any] 							=		[
														    torch.load(os.path.join(_lowerCAmelCase   ,      f"""consolidated.{i:02d}.pth"""  )   ,      map_location='''cpu'''  )
														    for i in range(_lowerCAmelCase  )
														]
							UpperCAmelCase :       Any 							=		0
							UpperCAmelCase :       str 							=		{'''weight_map''': {}}
							for layer_i in range(_lowerCAmelCase  ):
														UpperCAmelCase :       Optional[Any] 							=		f"""pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"""
														if model_size == "7B":
																					# Unsharded
																					UpperCAmelCase :       Optional[int] 							=		{
																					    f"""model.layers.{layer_i}.self_attn.q_proj.weight""": permute(
																					        loaded[f"""layers.{layer_i}.attention.wq.weight"""]  ),
																					    f"""model.layers.{layer_i}.self_attn.k_proj.weight""": permute(
																					        loaded[f"""layers.{layer_i}.attention.wk.weight"""]  ),
																					    f"""model.layers.{layer_i}.self_attn.v_proj.weight""": loaded[f"""layers.{layer_i}.attention.wv.weight"""],
																					    f"""model.layers.{layer_i}.self_attn.o_proj.weight""": loaded[f"""layers.{layer_i}.attention.wo.weight"""],
																					    f"""model.layers.{layer_i}.mlp.gate_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w1.weight"""],
																					    f"""model.layers.{layer_i}.mlp.down_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w2.weight"""],
																					    f"""model.layers.{layer_i}.mlp.up_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w3.weight"""],
																					    f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[f"""layers.{layer_i}.attention_norm.weight"""],
																					    f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[f"""layers.{layer_i}.ffn_norm.weight"""],
																					}
														else:
																					# Sharded
																					# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
																					# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
																					# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

																					UpperCAmelCase :       List[str] 							=		{
																					    f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[0][
																					        f"""layers.{layer_i}.attention_norm.weight"""
																					    ].clone(),
																					    f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[0][
																					        f"""layers.{layer_i}.ffn_norm.weight"""
																					    ].clone(),
																					}
																					UpperCAmelCase :       Union[str, Any] 							=		permute(
																					    torch.cat(
																					        [
																					            loaded[i][f"""layers.{layer_i}.attention.wq.weight"""].view(_lowerCAmelCase   ,      _lowerCAmelCase   ,      _lowerCAmelCase  )
																					            for i in range(_lowerCAmelCase  )
																					        ]   ,      dim=0   ,      ).reshape(_lowerCAmelCase   ,      _lowerCAmelCase  )  )
																					UpperCAmelCase :       Optional[Any] 							=		permute(
																					    torch.cat(
																					        [
																					            loaded[i][f"""layers.{layer_i}.attention.wk.weight"""].view(
																					                _lowerCAmelCase   ,      _lowerCAmelCase   ,      _lowerCAmelCase  )
																					            for i in range(_lowerCAmelCase  )
																					        ]   ,      dim=0   ,      ).reshape(_lowerCAmelCase   ,      _lowerCAmelCase  )   ,      _lowerCAmelCase   ,      _lowerCAmelCase   ,      _lowerCAmelCase   ,      )
																					UpperCAmelCase :       str 							=		torch.cat(
																					    [
																					        loaded[i][f"""layers.{layer_i}.attention.wv.weight"""].view(
																					            _lowerCAmelCase   ,      _lowerCAmelCase   ,      _lowerCAmelCase  )
																					        for i in range(_lowerCAmelCase  )
																					    ]   ,      dim=0   ,      ).reshape(_lowerCAmelCase   ,      _lowerCAmelCase  )

																					UpperCAmelCase :       Optional[int] 							=		torch.cat(
																					    [loaded[i][f"""layers.{layer_i}.attention.wo.weight"""] for i in range(_lowerCAmelCase  )]   ,      dim=1  )
																					UpperCAmelCase :       Any 							=		torch.cat(
																					    [loaded[i][f"""layers.{layer_i}.feed_forward.w1.weight"""] for i in range(_lowerCAmelCase  )]   ,      dim=0  )
																					UpperCAmelCase :       str 							=		torch.cat(
																					    [loaded[i][f"""layers.{layer_i}.feed_forward.w2.weight"""] for i in range(_lowerCAmelCase  )]   ,      dim=1  )
																					UpperCAmelCase :       Tuple 							=		torch.cat(
																					    [loaded[i][f"""layers.{layer_i}.feed_forward.w3.weight"""] for i in range(_lowerCAmelCase  )]   ,      dim=0  )

														UpperCAmelCase :       Any 							=		inv_freq
														for k, v in state_dict.items():
																					UpperCAmelCase :       List[Any] 							=		filename
																					param_count += v.numel()
														torch.save(_lowerCAmelCase   ,      os.path.join(_lowerCAmelCase   ,      _lowerCAmelCase  )  )

							UpperCAmelCase :       Optional[int] 							=		f"""pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"""
							if model_size == "7B":
														# Unsharded
														UpperCAmelCase :       str 							=		{
														    '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
														    '''model.norm.weight''': loaded['''norm.weight'''],
														    '''lm_head.weight''': loaded['''output.weight'''],
														}
							else:
														UpperCAmelCase :       Any 							=		{
														    '''model.norm.weight''': loaded[0]['''norm.weight'''],
														    '''model.embed_tokens.weight''': torch.cat(
														        [loaded[i]['''tok_embeddings.weight'''] for i in range(_lowerCAmelCase  )]   ,      dim=1  ),
														    '''lm_head.weight''': torch.cat([loaded[i]['''output.weight'''] for i in range(_lowerCAmelCase  )]   ,      dim=0  ),
														}

							for k, v in state_dict.items():
														UpperCAmelCase :       Optional[int] 							=		filename
														param_count += v.numel()
							torch.save(_lowerCAmelCase   ,      os.path.join(_lowerCAmelCase   ,      _lowerCAmelCase  )  )

							# Write configs
							UpperCAmelCase :       Union[str, Any] 							=		{'''total_size''': param_count * 2}
							write_json(_lowerCAmelCase   ,      os.path.join(_lowerCAmelCase   ,      '''pytorch_model.bin.index.json'''  )  )
							UpperCAmelCase :       int 							=		params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
							UpperCAmelCase :       Tuple 							=		params['''multiple_of'''] if '''multiple_of''' in params else 256
							UpperCAmelCase :       Any 							=		LlamaConfig(
							    hidden_size=_lowerCAmelCase   ,      intermediate_size=compute_intermediate_size(_lowerCAmelCase   ,      _lowerCAmelCase   ,      _lowerCAmelCase  )   ,      num_attention_heads=params['''n_heads''']   ,      num_hidden_layers=params['''n_layers''']   ,      rms_norm_eps=params['''norm_eps''']   ,      num_key_value_heads=_lowerCAmelCase   ,      )
							config.save_pretrained(_lowerCAmelCase  )

							# Make space so we can load the model properly now.
							del state_dict
							del loaded
							gc.collect()

							print('''Loading the checkpoint in a Llama model.'''  )
							UpperCAmelCase :       Optional[int] 							=		LlamaForCausalLM.from_pretrained(_lowerCAmelCase   ,      torch_dtype=torch.floataa   ,      low_cpu_mem_usage=_lowerCAmelCase  )
							# Avoid saving this as part of the config.
							del model.config._name_or_path

							print('''Saving in the Transformers format.'''  )
							model.save_pretrained(_lowerCAmelCase   ,      safe_serialization=_lowerCAmelCase  )
							shutil.rmtree(_lowerCAmelCase  )


def  snake_case_     (					_lowerCAmelCase  :						int   ,      _lowerCAmelCase  :						Union[str, Any]  )      ->   List[str]:
							# Initialize the tokenizer based on the `spm` model
							UpperCAmelCase :       Dict 							=		LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
							print(f"""Saving a {tokenizer_class.__name__} to {tokenizer_path}."""  )
							UpperCAmelCase :       List[Any] 							=		tokenizer_class(_lowerCAmelCase  )
							tokenizer.save_pretrained(_lowerCAmelCase  )


def  snake_case_     (					)      ->   List[Any]:
							UpperCAmelCase :       int 							=		argparse.ArgumentParser()
							parser.add_argument(
							    '''--input_dir'''   ,      help='''Location of LLaMA weights, which contains tokenizer.model and model folders'''   ,      )
							parser.add_argument(
							    '''--model_size'''   ,      choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only''']   ,      )
							parser.add_argument(
							    '''--output_dir'''   ,      help='''Location to write HF model and tokenizer'''   ,      )
							parser.add_argument('''--safe_serialization'''   ,      type=_lowerCAmelCase   ,      help='''Whether or not to save using `safetensors`.'''  )
							UpperCAmelCase :       List[Any] 							=		parser.parse_args()
							if args.model_size != "tokenizer_only":
														write_model(
														    model_path=args.output_dir   ,      input_base_path=os.path.join(args.input_dir   ,      args.model_size  )   ,      model_size=args.model_size   ,      safe_serialization=args.safe_serialization   ,      )
							UpperCAmelCase :       Optional[int] 							=		os.path.join(args.input_dir   ,      '''tokenizer.model'''  )
							write_tokenizer(args.output_dir   ,      _lowerCAmelCase  )


if __name__ == "__main__":
						main()





