
import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
      from transformers import LlamaTokenizerFast
except ImportError as e:
      warnings.warn(e)
      warnings.warn(
          '''The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'''
      )
      a__							:	List[Any]  						=			None
a__							:	List[Any]  						=			{
    '''7B''': 11_008,
    '''13B''': 13_824,
    '''30B''': 17_920,
    '''65B''': 22_016,
    '''70B''': 28_672,
}
a__							:	Any  						=			{
    '''7B''': 1,
    '''7Bf''': 1,
    '''13B''': 2,
    '''13Bf''': 2,
    '''30B''': 4,
    '''65B''': 8,
    '''70B''': 8,
    '''70Bf''': 8,
}







def 							UpperCAmelCase_(  a__      ,		a__=1      ,		a__=256					):



    """simple docstring"""



    return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3					)					) + multiple_of - 1) // multiple_of)







def 							UpperCAmelCase_(  a__					):



    """simple docstring"""



    with open(snake_case_      ,		'''r'''					) as f:
        return json.load(snake_case_					)







def 							UpperCAmelCase_(  a__      ,		a__					):



    """simple docstring"""



    with open(snake_case_      ,		'''w'''					) as f:
        json.dump(snake_case_      ,		snake_case_					)







def 							UpperCAmelCase_(  a__      ,		a__      ,		a__      ,		a__=True					):



    """simple docstring"""



    os.makedirs(snake_case_      ,		exist_ok=snake_case_					)
    SCREAMING_SNAKE_CASE							:							Tuple								=      os.path.join(snake_case_      ,		'''tmp'''					)
    os.makedirs(snake_case_      ,		exist_ok=snake_case_					)

    SCREAMING_SNAKE_CASE							:							int								=      read_json(os.path.join(snake_case_      ,		'''params.json'''					)					)
    SCREAMING_SNAKE_CASE							:							Any								=      NUM_SHARDS[model_size]
    SCREAMING_SNAKE_CASE							:							Dict								=      params["""n_layers"""]
    SCREAMING_SNAKE_CASE							:							Optional[Any]								=      params["""n_heads"""]
    SCREAMING_SNAKE_CASE							:							Union[str, Any]								=      n_heads // num_shards
    SCREAMING_SNAKE_CASE							:							Dict								=      params["""dim"""]
    SCREAMING_SNAKE_CASE							:							Optional[Any]								=      dim // n_heads
    SCREAMING_SNAKE_CASE							:							List[Any]								=      10_000.0
    SCREAMING_SNAKE_CASE							:							List[str]								=      1.0 / (base ** (torch.arange(0      ,		snake_case_      ,		2					).float() / dims_per_head))

    if "n_kv_heads" in params:
        SCREAMING_SNAKE_CASE							:							str								=      params["""n_kv_heads"""]  # for GQA / MQA
        SCREAMING_SNAKE_CASE							:							List[Any]								=      n_heads_per_shard // num_key_value_heads
        SCREAMING_SNAKE_CASE							:							List[str]								=      dim // num_key_value_heads
    else:  # compatibility with other checkpoints
        SCREAMING_SNAKE_CASE							:							Any								=      n_heads
        SCREAMING_SNAKE_CASE							:							Optional[int]								=      n_heads_per_shard
        SCREAMING_SNAKE_CASE							:							List[Any]								=      dim

    # permute for sliced rotary
    def permute(a__      ,		a__=n_heads      ,		a__=dim      ,		a__=dim					):
        return w.view(snake_case_      ,		dima // n_heads // 2      ,		2      ,		snake_case_					).transpose(1      ,		2					).reshape(snake_case_      ,		snake_case_					)

    print(F"""Fetching all parameters from the checkpoint at {input_base_path}."""					)
    # Load weights
    if model_size == "7B":
        # Not sharded
        # (The sharded implementation would also work, but this is simpler.)
        SCREAMING_SNAKE_CASE							:							str								=      torch.load(os.path.join(snake_case_      ,		'''consolidated.00.pth'''					)      ,		map_location='''cpu'''					)
    else:
        # Sharded
        SCREAMING_SNAKE_CASE							:							int								=      [
            torch.load(os.path.join(snake_case_      ,		F"""consolidated.{i:02d}.pth"""					)      ,		map_location='''cpu'''					)
            for i in range(snake_case_					)
        ]
    SCREAMING_SNAKE_CASE							:							Optional[int]								=      0
    SCREAMING_SNAKE_CASE							:							Optional[Any]								=      {"""weight_map""": {}}
    for layer_i in range(snake_case_					):
        SCREAMING_SNAKE_CASE							:							int								=      F"""pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"""
        if model_size == "7B":
            # Unsharded
            SCREAMING_SNAKE_CASE							:							Union[str, Any]								=      {
                F"""model.layers.{layer_i}.self_attn.q_proj.weight""": permute(
                    loaded[F"""layers.{layer_i}.attention.wq.weight"""]					),
                F"""model.layers.{layer_i}.self_attn.k_proj.weight""": permute(
                    loaded[F"""layers.{layer_i}.attention.wk.weight"""]					),
                F"""model.layers.{layer_i}.self_attn.v_proj.weight""": loaded[F"""layers.{layer_i}.attention.wv.weight"""],
                F"""model.layers.{layer_i}.self_attn.o_proj.weight""": loaded[F"""layers.{layer_i}.attention.wo.weight"""],
                F"""model.layers.{layer_i}.mlp.gate_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w1.weight"""],
                F"""model.layers.{layer_i}.mlp.down_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w2.weight"""],
                F"""model.layers.{layer_i}.mlp.up_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w3.weight"""],
                F"""model.layers.{layer_i}.input_layernorm.weight""": loaded[F"""layers.{layer_i}.attention_norm.weight"""],
                F"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[F"""layers.{layer_i}.ffn_norm.weight"""],
            }
        else:
            # Sharded
            # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
            # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
            # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

            SCREAMING_SNAKE_CASE							:							Any								=      {
                F"""model.layers.{layer_i}.input_layernorm.weight""": loaded[0][
                    F"""layers.{layer_i}.attention_norm.weight"""
                ].clone(),
                F"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[0][
                    F"""layers.{layer_i}.ffn_norm.weight"""
                ].clone(),
            }
            SCREAMING_SNAKE_CASE							:							Union[str, Any]								=      permute(
                torch.cat(
                    [
                        loaded[i][F"""layers.{layer_i}.attention.wq.weight"""].view(snake_case_      ,		snake_case_      ,		snake_case_					)
                        for i in range(snake_case_					)
                    ]      ,		dim=0      ,		).reshape(snake_case_      ,		snake_case_					)					)
            SCREAMING_SNAKE_CASE							:							Optional[Any]								=      permute(
                torch.cat(
                    [
                        loaded[i][F"""layers.{layer_i}.attention.wk.weight"""].view(
                            snake_case_      ,		snake_case_      ,		snake_case_					)
                        for i in range(snake_case_					)
                    ]      ,		dim=0      ,		).reshape(snake_case_      ,		snake_case_					)      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		)
            SCREAMING_SNAKE_CASE							:							Union[str, Any]								=      torch.cat(
                [
                    loaded[i][F"""layers.{layer_i}.attention.wv.weight"""].view(
                        snake_case_      ,		snake_case_      ,		snake_case_					)
                    for i in range(snake_case_					)
                ]      ,		dim=0      ,		).reshape(snake_case_      ,		snake_case_					)

            SCREAMING_SNAKE_CASE							:							str								=      torch.cat(
                [loaded[i][F"""layers.{layer_i}.attention.wo.weight"""] for i in range(snake_case_					)]      ,		dim=1					)
            SCREAMING_SNAKE_CASE							:							Tuple								=      torch.cat(
                [loaded[i][F"""layers.{layer_i}.feed_forward.w1.weight"""] for i in range(snake_case_					)]      ,		dim=0					)
            SCREAMING_SNAKE_CASE							:							Dict								=      torch.cat(
                [loaded[i][F"""layers.{layer_i}.feed_forward.w2.weight"""] for i in range(snake_case_					)]      ,		dim=1					)
            SCREAMING_SNAKE_CASE							:							Any								=      torch.cat(
                [loaded[i][F"""layers.{layer_i}.feed_forward.w3.weight"""] for i in range(snake_case_					)]      ,		dim=0					)

        SCREAMING_SNAKE_CASE							:							Any								=      inv_freq
        for k, v in state_dict.items():
            SCREAMING_SNAKE_CASE							:							Dict								=      filename
            param_count += v.numel()
        torch.save(snake_case_      ,		os.path.join(snake_case_      ,		snake_case_					)					)

    SCREAMING_SNAKE_CASE							:							Optional[Any]								=      F"""pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"""
    if model_size == "7B":
        # Unsharded
        SCREAMING_SNAKE_CASE							:							Optional[Any]								=      {
            """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
            """model.norm.weight""": loaded["""norm.weight"""],
            """lm_head.weight""": loaded["""output.weight"""],
        }
    else:
        SCREAMING_SNAKE_CASE							:							Optional[int]								=      {
            """model.norm.weight""": loaded[0]["""norm.weight"""],
            """model.embed_tokens.weight""": torch.cat(
                [loaded[i]['''tok_embeddings.weight'''] for i in range(snake_case_					)]      ,		dim=1					),
            """lm_head.weight""": torch.cat([loaded[i]['''output.weight'''] for i in range(snake_case_					)]      ,		dim=0					),
        }

    for k, v in state_dict.items():
        SCREAMING_SNAKE_CASE							:							List[str]								=      filename
        param_count += v.numel()
    torch.save(snake_case_      ,		os.path.join(snake_case_      ,		snake_case_					)					)

    # Write configs
    SCREAMING_SNAKE_CASE							:							Tuple								=      {"""total_size""": param_count * 2}
    write_json(snake_case_      ,		os.path.join(snake_case_      ,		'''pytorch_model.bin.index.json'''					)					)
    SCREAMING_SNAKE_CASE							:							Any								=      params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
    SCREAMING_SNAKE_CASE							:							int								=      params["""multiple_of"""] if """multiple_of""" in params else 256
    SCREAMING_SNAKE_CASE							:							str								=      LlamaConfig(
        hidden_size=snake_case_      ,		intermediate_size=compute_intermediate_size(snake_case_      ,		snake_case_      ,		snake_case_					)      ,		num_attention_heads=params['''n_heads''']      ,		num_hidden_layers=params['''n_layers''']      ,		rms_norm_eps=params['''norm_eps''']      ,		num_key_value_heads=snake_case_      ,		)
    config.save_pretrained(snake_case_					)

    # Make space so we can load the model properly now.
    del state_dict
    del loaded
    gc.collect()

    print('''Loading the checkpoint in a Llama model.'''					)
    SCREAMING_SNAKE_CASE							:							Optional[Any]								=      LlamaForCausalLM.from_pretrained(snake_case_      ,		torch_dtype=torch.floataa      ,		low_cpu_mem_usage=snake_case_					)
    # Avoid saving this as part of the config.
    del model.config._name_or_path

    print('''Saving in the Transformers format.'''					)
    model.save_pretrained(snake_case_      ,		safe_serialization=snake_case_					)
    shutil.rmtree(snake_case_					)







def 							UpperCAmelCase_(  a__      ,		a__					):



    """simple docstring"""



    SCREAMING_SNAKE_CASE							:							Union[str, Any]								=      LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
    print(F"""Saving a {tokenizer_class.__name__} to {tokenizer_path}."""					)
    SCREAMING_SNAKE_CASE							:							List[Any]								=      tokenizer_class(snake_case_					)
    tokenizer.save_pretrained(snake_case_					)







def 							UpperCAmelCase_(  ):



    """simple docstring"""



    SCREAMING_SNAKE_CASE							:							Dict								=      argparse.ArgumentParser()
    parser.add_argument(
        '''--input_dir'''      ,		help='''Location of LLaMA weights, which contains tokenizer.model and model folders'''      ,		)
    parser.add_argument(
        '''--model_size'''      ,		choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only''']      ,		)
    parser.add_argument(
        '''--output_dir'''      ,		help='''Location to write HF model and tokenizer'''      ,		)
    parser.add_argument('''--safe_serialization'''      ,		type=snake_case_      ,		help='''Whether or not to save using `safetensors`.'''					)
    SCREAMING_SNAKE_CASE							:							Union[str, Any]								=      parser.parse_args()
    if args.model_size != "tokenizer_only":
        write_model(
            model_path=args.output_dir      ,		input_base_path=os.path.join(args.input_dir      ,		args.model_size					)      ,		model_size=args.model_size      ,		safe_serialization=args.safe_serialization      ,		)
    SCREAMING_SNAKE_CASE							:							int								=      os.path.join(args.input_dir      ,		'''tokenizer.model'''					)
    write_tokenizer(args.output_dir      ,		snake_case_					)


if __name__ == "__main__":
      main()






from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy







@add_start_docstrings(
    "The RoBERTa Model transformer with early exiting (DeeRoBERTa). ",UpperCamelCase__,)
class 						lowercase							(   UpperCamelCase__		):
     _a							 =						RobertaConfig
     _a							 =						"roberta"

     def __init__(  self		,     _a				)						->							Optional[int]:
           super().__init__(_a				)

           _A							:     Union[str, Any]			=						RobertaEmbeddings(_a				)
           self.init_weights()







@add_start_docstrings(
    "RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,\n    also takes care of multi-layer training. ",UpperCamelCase__,)
class 						lowercase							(   UpperCamelCase__		):
     _a							 =						RobertaConfig
     _a							 =						"roberta"

     def __init__(  self		,     _a				)						->							str:
           super().__init__(_a				)
           _A							:     Any			=						config.num_labels
           _A							:     Dict			=						config.num_hidden_layers

           _A							:     List[str]			=						DeeRobertaModel(_a				)
           _A							:     int			=						nn.Dropout(config.hidden_dropout_prob				)
           _A							:     int			=						nn.Linear(config.hidden_size		,     self.config.num_labels				)



     @add_start_docstrings_to_model_forward(_a				)
     def 					a__	(  self		,     _a=None		,     _a=None		,     _a=None		,     _a=None		,     _a=None		,     _a=None		,     _a=None		,     _a=-1		,     _a=False		,     )						->							Any:
           _A							:     Optional[int]			=						self.num_layers
           try:
                 _A							:     List[str]			=						self.roberta(
                     _a		,     attention_mask=_a		,     token_type_ids=_a		,     position_ids=_a		,     head_mask=_a		,     inputs_embeds=_a		,     )

                 _A							:     List[str]			=						outputs[1]

                 _A							:     List[str]			=						self.dropout(_a				)
                 _A							:     Optional[Any]			=						self.classifier(_a				)
                 _A							:     List[Any]			=						(logits,) + outputs[2:]  # add hidden states and attention if they are here
           except HighwayException as e:
                 _A							:     List[Any]			=						e.message
                 _A							:     Optional[int]			=						e.exit_layer
                 _A							:     Optional[int]			=						outputs[0]

           if not self.training:
                 _A							:     int			=						entropy(_a				)
                 _A							:     int			=						[]
                 _A							:     int			=						[]
           if labels is not None:
                 if self.num_labels == 1:
                       #  We are doing regression
                       _A							:     Union[str, Any]			=						MSELoss()
                       _A							:     Tuple			=						loss_fct(logits.view(-1				)		,     labels.view(-1				)				)
                 else:
                       _A							:     List[Any]			=						CrossEntropyLoss()
                       _A							:     Dict			=						loss_fct(logits.view(-1		,     self.num_labels				)		,     labels.view(-1				)				)

                 # work with highway exits
                 _A							:     Optional[Any]			=						[]
                 for highway_exit in outputs[-1]:
                       _A							:     Tuple			=						highway_exit[0]
                       if not self.training:
                             highway_logits_all.append(_a				)
                             highway_entropy.append(highway_exit[2]				)
                       if self.num_labels == 1:
                             #  We are doing regression
                             _A							:     List[str]			=						MSELoss()
                             _A							:     Optional[int]			=						loss_fct(highway_logits.view(-1				)		,     labels.view(-1				)				)
                       else:
                             _A							:     List[Any]			=						CrossEntropyLoss()
                             _A							:     Tuple			=						loss_fct(highway_logits.view(-1		,     self.num_labels				)		,     labels.view(-1				)				)
                       highway_losses.append(_a				)

                 if train_highway:
                       _A							:     Dict			=						(sum(highway_losses[:-1]				),) + outputs
                       # exclude the final highway, of course
                 else:
                       _A							:     int			=						(loss,) + outputs
           if not self.training:
                 _A							:     Optional[Any]			=						outputs + ((original_entropy, highway_entropy), exit_layer)
                 if output_layer >= 0:
                       _A							:     Union[str, Any]			=						(
                           (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
                       )  # use the highway of the last layer

           return outputs  # (loss), logits, (hidden_states), (attentions), entropy

