


import argparse

# utilities to import the model weights and config file
import os
import pickle as pkl

# PyTorch + new model classes
import torch
from torch import nn

from transformers import AutoTokenizer, MegaConfig, MegaForMaskedLM


# import the EncoderLayer class used to pretrain
# !! NOTE !! this requires the version of fairseq that is built when you install the Mega source
try:
       from fairseq.modules.mega_layer import MegaEncoderLayer
except ImportError:
       raise ImportError('You need to install the version of fairseq from the Mega repo!')





class 			lowerCAmelCase__(     nn.Module    ):






    '''simple docstring'''





    def __init__(						self					: Union[str, Any]							, __snake_case					: int							, __snake_case					: Any							, __snake_case					: Dict						):





       '''simple docstring'''
       super().__init__()
       UpperCAmelCase_				:   List[Any]							= mega_args
       UpperCAmelCase_				:   str							= nn.Embedding(__snake_case							, self.mega_args.encoder_embed_dim						)
       UpperCAmelCase_				:   Tuple							= nn.ModuleList([MegaEncoderLayer(self.mega_args						) for _ in range(__snake_case						)]						)
       UpperCAmelCase_				:   Optional[int]							= depth





    def        _lowerCamelCase			(						self					: Optional[int]							, __snake_case					: List[Any]							, __snake_case					: int							, __snake_case					: Optional[int]=True							, __snake_case					: Optional[Any]=0						):





       '''simple docstring'''

       # Mega expects embeddings to be (time, batch, embedding size), but
       # Hugging Face returns tokens as (batch, time)
       if batch_first:
          UpperCAmelCase_				:   List[str]							= input_ids.T

       # to make things more confusing, Mega expects the attention mask to
       # be (batch, time), but with values of 0 (normal token) and 1 (ignore token)
       # which is the opposite of what HF returns
       if ignore_mask_value == 0:
          UpperCAmelCase_				:   Optional[Any]							= 1 - attention_mask

       # get token embeddings from IDs
       UpperCAmelCase_				:   Any							= self.embedding_layer(__snake_case						)

       # pass through the Mega layers
       # input is (time, batch, encoder dim) and output is the same
       for encoder in self.encoders:
          UpperCAmelCase_				:   Any							= encoder(__snake_case							, __snake_case						)

       # return according to the shape specified
       if batch_first:
          # (T, B, H) --> (B, T, H)
          return torch.transpose(__snake_case							, 0							, 1						)
       else:
          return embeds





class 			lowerCAmelCase__(     nn.Module    ):






    '''simple docstring'''





    def __init__(						self					: str							, __snake_case					: Tuple							, __snake_case					: Union[str, Any]							, __snake_case					: Dict						):





       '''simple docstring'''
       super().__init__()
       UpperCAmelCase_				:   Any							= MegaLM(__snake_case							, __snake_case							, __snake_case						)
       UpperCAmelCase_				:   int							= nn.Linear(mega_args.encoder_embed_dim							, __snake_case						)
       UpperCAmelCase_				:   Tuple							= nn.Dropout(p=0.1						)





    def        _lowerCamelCase			(						self					: Any							, __snake_case					: List[str]							, __snake_case					: Dict							, __snake_case					: Any=True							, __snake_case					: int=0						):





       '''simple docstring'''
       UpperCAmelCase_				:   List[str]							= self.mega(__snake_case							, __snake_case							, __snake_case							, __snake_case						)
       return self.mlm_head(self.dropout(__snake_case						)						)





def   snake_case_    (			__lowercase      ,							__lowercase      ,							__lowercase							):






   with open(os.path.join(_lowercase      ,							'''model_args.pkl'''							)      ,							'''rb'''							) as f:
      UpperCAmelCase_				:   Any							= pkl.load(_lowercase							)

   # load the original encoder
   UpperCAmelCase_				:   int							= OriginalMegaForMaskedLM(**_lowercase							).eval()

   # load its weights
   print(
       '''Original Mega encoder:'''      ,							original_mlm.mega.load_state_dict(
           torch.load(os.path.join(_lowercase      ,							'''encoder_weights.pt'''							)      ,							map_location='''cpu'''							)							)      ,							)
   print(
       '''Original Mega MLM layer:'''      ,							original_mlm.mlm_head.load_state_dict(
           torch.load(os.path.join(_lowercase      ,							'''mlm_head_weights.pt'''							)      ,							map_location='''cpu'''							)							)      ,							)

   # create a new config from the old one
   UpperCAmelCase_				:   Tuple							= MegaConfig(
       num_hidden_layers=mega_original_args['''depth''']      ,							vocab_size=mega_original_args['''vocab_size''']      ,							hidden_size=mega_original_args['''mega_args'''].encoder_embed_dim      ,							shared_representation_size=mega_original_args['''mega_args'''].encoder_z_dim      ,							intermediate_size=mega_original_args['''mega_args'''].encoder_hidden_dim      ,							ema_projection_size=mega_original_args['''mega_args'''].encoder_n_dim      ,							dropout_prob=mega_original_args['''mega_args'''].dropout      ,							attention_probs_dropout_prob=mega_original_args['''mega_args'''].attention_dropout      ,							hidden_dropout_prob=mega_original_args['''mega_args'''].hidden_dropout      ,							activation=mega_original_args['''mega_args'''].activation_fn      ,							attention_activation=mega_original_args['''mega_args'''].attention_activation_fn      ,							bidirectional=mega_original_args['''mega_args'''].bidirectional      ,							use_chunking=mega_original_args['''mega_args'''].encoder_chunk_size > 0      ,							chunk_size=mega_original_args['''mega_args'''].encoder_chunk_size      ,							truncation=mega_original_args['''mega_args'''].truncation_length      ,							normalization_type=mega_original_args['''mega_args'''].normalization_type      ,							normalize_before_mega=_lowercase      ,							norm_affine=_lowercase      ,							use_feature_dropout=mega_original_args['''mega_args'''].feature_dropout      ,							relative_positional_bias=mega_original_args['''mega_args'''].rel_pos_bias      ,							max_positions=mega_original_args['''mega_args'''].max_source_positions      ,							nffn_hidden_size=mega_original_args['''mega_args'''].encoder_ffn_embed_dim      ,							normalize_before_ffn=mega_original_args['''mega_args'''].normalize_before      ,							nffn_activation_dropout_prob=0.0      ,							add_token_type_embeddings=_lowercase      ,							add_lm_hidden_dense_layer=_lowercase      ,							)

   UpperCAmelCase_				:   List[Any]							= MegaForMaskedLM(_lowercase							).eval()

   # the originl checkpoint just uses nn.Embedding for the word embeddings
   # we use a wrapper module for embeddings to add support for positional embeddings
   UpperCAmelCase_				:   Tuple							= original_mlm.mega.embedding_layer.weight

   # modify the state dictionary of the original checkpoint to account for naming issues in the Hugging Face
   # ecosystem -- any names containing "beta" or "gamma" aren't safe to use and are renamed upon _load_pretrained,
   # also renaming previously confusing parameter names
   UpperCAmelCase_				:   List[str]							= original_mlm.mega.encoders.state_dict()
   UpperCAmelCase_				:   Any							= {}
   for module_name in original_state_dict.keys():
      UpperCAmelCase_				:   int							= None
      # have to handle gamma, beta, and alpha differently due to their use
      # in multiple modules within the original repository;
      # beta is used in EMA, MovingAverageGatedAttention, and RotaryRelativePositionalBias, and must be renamed due to flax/tf weights
      # the EMA sublayer was renamed from "move" to "ema_gate" for readability, so that is also done here
      if "beta" in module_name:
         # EMA sub-layers were always called "move" in the original repo
         if "move.beta" in module_name:
            UpperCAmelCase_				:   Dict							= module_name.replace('''move.beta'''      ,							'''ema_gate.ema_expansion_matrix'''							)
         elif "mega_layer.beta" in module_name:
            UpperCAmelCase_				:   Optional[Any]							= module_name.replace('''beta'''      ,							'''qk_bias'''							)
         else:
            UpperCAmelCase_				:   Tuple							= module_name.replace('''beta'''      ,							'''b_param'''							)
        # beta is used in EMA and MovingAverageGatedAttention, and must be renamed due to flax/tf weights
      elif "gamma" in module_name:
         if "move.gamma" in module_name:
            UpperCAmelCase_				:   List[str]							= module_name.replace('''move.gamma'''      ,							'''ema_gate.kernel_projection_matrix'''							)
         elif "mega_layer.gamma" in module_name:
            UpperCAmelCase_				:   Union[str, Any]							= module_name.replace('''gamma'''      ,							'''qk_weight'''							)
         else:
            UpperCAmelCase_				:   Union[str, Any]							= module_name.replace('''gamma'''      ,							'''g_param'''							)
        # alpha is used in EMA and positional bias; renaming to improve readability
      elif "move.alpha" in module_name:
         UpperCAmelCase_				:   Dict							= module_name.replace('''move.alpha'''      ,							'''ema_gate.decay_factor'''							)
      # delta is only used in EMA; renaming to improve readability
      elif "move.delta" in module_name:
         UpperCAmelCase_				:   Dict							= module_name.replace('''move.delta'''      ,							'''ema_gate.damping_factor'''							)
      # omega is only used in EMA; renaming to improve readability
      elif "omega" in module_name:
         UpperCAmelCase_				:   Optional[int]							= module_name.replace('''move.omega'''      ,							'''ema_gate.residual_weight'''							)

      if new_module_name:
         UpperCAmelCase_				:   Any							= new_module_name

   if len(_lowercase							) != 0:
      print(F'''Renaming these keys: {updated_keys.keys()}'''							)
   else:
      print('''No need to rename state dict entries'''							)
   for old, new in updated_keys.items():
      UpperCAmelCase_				:   List[Any]							= original_state_dict.pop(_lowercase							)

   # now attempt to load the state dictionary with updated names
   # note that we now call it `mega.layers` instead of `mega.encoders` due to hugging face style
   print('''HF Mega encoder:'''      ,							hf_mlm.mega.layers.load_state_dict(_lowercase							)							)

   # load the MLM head weights directly
   print(
       '''HF Mega MLM layer:'''      ,							hf_mlm.mlm_head.load_state_dict(
           torch.load(os.path.join(_lowercase      ,							'''mlm_head_weights.pt'''							)      ,							map_location='''cpu'''							)							)      ,							)

   # test on a randomly generated input sequence
   UpperCAmelCase_				:   str							= torch.randint(0      ,							hf_config.vocab_size      ,							size=(4, 2_5_6)							)
   UpperCAmelCase_				:   List[str]							= torch.ones_like(_lowercase							)
   # mask a few tokens to make sure masking is applied appropriately :)
   UpperCAmelCase_				:   Any							= 0

   # run forward passes
   UpperCAmelCase_				:   Tuple							= original_mlm(_lowercase      ,							_lowercase      ,							batch_first=_lowercase      ,							ignore_mask_value=0							)
   UpperCAmelCase_				:   Optional[Any]							= hf_mlm(_lowercase      ,							_lowercase							)[0]

   # print shapes and diff
   print(F'''original output {original_output.shape}'''							)
   print(F'''hf output {hf_output.shape}'''							)
   print(F'''max diff: {(original_output - hf_output).max()}'''							)  # 0.0
   UpperCAmelCase_				:   Union[str, Any]							= torch.allclose(_lowercase      ,							_lowercase      ,							atol=1e-3							)

   if success:
      print('''Yay!'''							)
      hf_mlm.save_pretrained(_lowercase							)
   else:
      raise RuntimeError(F'''Something\'s broken :(\nOriginal:\n{original_output}\n\nHF\n{hf_output}\n{hf_mlm}'''							)

   if includes_tokenizer:
      print('''Transferring tokenizer'''							)
      UpperCAmelCase_				:   List[str]							= AutoTokenizer.from_pretrained(_lowercase							)
      tokenizer.save_pretrained(_lowercase							)


if __name__ == "__main__":
       __UpperCamelCase					:	Optional[int]          =				argparse.ArgumentParser()

       parser.add_argument(
           '--pretrained_checkpoint_path',
           default=None,
           type=str,
           required=True,
           help='Point to the directory containing your model weights using the official Mega repo',
       )

       parser.add_argument(
           '--output_path', default=None, type=str, required=True, help='Location to save the Hugging Face version'
       )

       parser.add_argument(
           '--includes_tokenizer',
           action='store_true',
           help='Use this flag if there is a Hugging Face tokenizer in the original checkpoint repo',
       )

       __UpperCamelCase					:	Tuple          =				parser.parse_args()

       convert_checkpoint_to_huggingface(args.pretrained_checkpoint_path, args.output_path, args.includes_tokenizer)



'''simple docstring'''



import math
from decimal import Decimal, getcontext




def       _a      (		_lowercase   :      int = 4000000    ):





     '''simple docstring'''






     try:
          __UpperCAmelCase      :					List[str]    				=     int(_lowercase    )
     except (TypeError, ValueError):
          raise TypeError('''Parameter n must be int or castable to int.'''    )
     if n <= 0:
          raise ValueError('''Parameter n must be greater than or equal to one.'''    )
     __UpperCAmelCase      :					Tuple    				=     100
     __UpperCAmelCase      :					Optional[Any]    				=     (Decimal(5    ) ** Decimal(0.5    ) + 1) / Decimal(2    )

     __UpperCAmelCase      :					Tuple    				=     (math.floor(math.log(n * (phi + 2)     ,   _lowercase    ) - 1    ) // 3) * 3 + 2
     __UpperCAmelCase      :					List[Any]    				=     Decimal(round(phi ** Decimal(index + 1    )    )    ) / (phi + 2)
     __UpperCAmelCase      :					List[str]    				=     num // 2
     return int(_lowercase    )


if __name__ == "__main__":
      print(f"""{solution() = }""")