
import math
from typing import Optional

import torch
import torch.utils.checkpoint as checkpoint
from torch import nn

from ....modeling_utils import PreTrainedModel
from ....utils import add_start_docstrings, logging
from ...bert.modeling_bert import BertModel
from .configuration_retribert import RetriBertConfig


__magic_name__  :  int     			=   logging.get_logger(__name__)

__magic_name__  :  List[Any]     			=   [
    'yjernite/retribert-base-uncased',
    # See all RetriBert models at https://huggingface.co/models?filter=retribert
]


class 							lowerCamelCase       (		__snake_case					):



  """simple docstring"""





  lowerCAmelCase_								=		RetriBertConfig
  lowerCAmelCase_								=		None
  lowerCAmelCase_								=		"""retribert"""

  def 					lowercase_						(					self						,    __UpperCamelCase					):
      if isinstance(__UpperCamelCase						,    nn.Linear					):
          module.weight.data.normal_(mean=0.0						,    std=self.config.initializer_range					)
          if module.bias is not None:
              module.bias.data.zero_()
      elif isinstance(__UpperCamelCase						,    nn.Embedding					):
          module.weight.data.normal_(mean=0.0						,    std=self.config.initializer_range					)
          if module.padding_idx is not None:
              module.weight.data[module.padding_idx].zero_()
      elif isinstance(__UpperCamelCase						,    nn.LayerNorm					):
          module.bias.data.zero_()
          module.weight.data.fill_(1.0					)



__magic_name__  :  str     			=   r'\n\n    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the\n    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads\n    etc.)\n\n    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.\n    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage\n    and behavior.\n\n    Parameters:\n        config ([`RetriBertConfig`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.\n'


@add_start_docstrings(
    """Bert Based model to embed queries or document for document retrieval."""			,  __snake_case			,  )
class 							lowerCamelCase       (		__snake_case					):



  """simple docstring"""





  def __init__(					self						,    __UpperCamelCase					):
      super().__init__(__UpperCamelCase					)
      A_								=					config.projection_dim

      A_								=					BertModel(__UpperCamelCase					)
      A_								=					None if config.share_encoders else BertModel(__UpperCamelCase					)
      A_								=					nn.Dropout(config.hidden_dropout_prob					)
      A_								=					nn.Linear(config.hidden_size						,    config.projection_dim						,    bias=__UpperCamelCase					)
      A_								=					nn.Linear(config.hidden_size						,    config.projection_dim						,    bias=__UpperCamelCase					)

      A_								=					nn.CrossEntropyLoss(reduction="mean"					)

      # Initialize weights and apply final processing
      self.post_init()

  def 					lowercase_						(					self						,    __UpperCamelCase						,    __UpperCamelCase						,    __UpperCamelCase						,    __UpperCamelCase=-1						,    ):
      # reproduces BERT forward pass with checkpointing
      if checkpoint_batch_size < 0 or input_ids.shape[0] < checkpoint_batch_size:
          return sent_encoder(__UpperCamelCase						,    attention_mask=__UpperCamelCase					)[1]
      else:
          # prepare implicit variables
          A_								=					input_ids.device
          A_								=					input_ids.size()
          A_								=					torch.zeros(__UpperCamelCase						,    dtype=torch.long						,    device=__UpperCamelCase					)
          A_								=					[None] * sent_encoder.config.num_hidden_layers
          A_								=					sent_encoder.get_extended_attention_mask(
              __UpperCamelCase						,    __UpperCamelCase					)

          # define function for checkpointing
          def partial_encode(*__UpperCamelCase					):
              A_								=					sent_encoder.encoder(
                  inputs[0]						,    attention_mask=inputs[1]						,    head_mask=__UpperCamelCase						,    )
              A_								=					encoder_outputs[0]
              A_								=					sent_encoder.pooler(__UpperCamelCase					)
              return pooled_output

          # run embedding layer on everything at once
          A_								=					sent_encoder.embeddings(
              input_ids=__UpperCamelCase						,    position_ids=__UpperCamelCase						,    token_type_ids=__UpperCamelCase						,    inputs_embeds=__UpperCamelCase					)
          # run encoding and pooling on one mini-batch at a time
          A_								=					[]
          for b in range(math.ceil(input_ids.shape[0] / checkpoint_batch_size					)					):
              A_								=					embedding_output[b * checkpoint_batch_size : (b + 1) * checkpoint_batch_size]
              A_								=					extended_attention_mask[b * checkpoint_batch_size : (b + 1) * checkpoint_batch_size]
              A_								=					checkpoint.checkpoint(__UpperCamelCase						,    __UpperCamelCase						,    __UpperCamelCase					)
              pooled_output_list.append(__UpperCamelCase					)
          return torch.cat(__UpperCamelCase						,    dim=0					)

  def 					lowercase_						(					self						,    __UpperCamelCase						,    __UpperCamelCase=None						,    __UpperCamelCase=-1						,    ):
      A_								=					self.embed_sentences_checkpointed(
          __UpperCamelCase						,    __UpperCamelCase						,    self.bert_query						,    __UpperCamelCase						,    )
      return self.project_query(__UpperCamelCase					)

  def 					lowercase_						(					self						,    __UpperCamelCase						,    __UpperCamelCase=None						,    __UpperCamelCase=-1						,    ):
      A_								=					self.embed_sentences_checkpointed(
          __UpperCamelCase						,    __UpperCamelCase						,    self.bert_query if self.bert_doc is None else self.bert_doc						,    __UpperCamelCase						,    )
      return self.project_doc(__UpperCamelCase					)

  def 					lowercase_						(					self						,    __UpperCamelCase						,    __UpperCamelCase						,    __UpperCamelCase						,    __UpperCamelCase						,    __UpperCamelCase = -1						,    ):
      A_								=					input_ids_query.device
      A_								=					self.embed_questions(__UpperCamelCase						,    __UpperCamelCase						,    __UpperCamelCase					)
      A_								=					self.embed_answers(__UpperCamelCase						,    __UpperCamelCase						,    __UpperCamelCase					)
      A_								=					torch.mm(__UpperCamelCase						,    a_reps.t()					)
      A_								=					self.ce_loss(__UpperCamelCase						,    torch.arange(compare_scores.shape[1]					).to(__UpperCamelCase					)					)
      A_								=					self.ce_loss(compare_scores.t()						,    torch.arange(compare_scores.shape[0]					).to(__UpperCamelCase					)					)
      A_								=					(loss_qa + loss_aq) / 2
      return loss








import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig





def        lowerCAmelCase       (				snake_case__   :      List[str]				,					snake_case__   :      List[str]				,					snake_case__   :      Optional[int]=0 )->			Union[str, Any]:
    # Format the message.
    if name is None:
        A_								=					None
    else:
        A_								=					"." * max(0				,					spaces - 2 ) + "# {:" + str(50 - spaces ) + "s}"
        A_								=					fmt.format(snake_case__ )

    # Print and recurse (if needed).
    if isinstance(snake_case__				,					snake_case__ ):
        if msg is not None:
            print(snake_case__ )
        for k in val.keys():
            recursive_print(snake_case__				,					val[k]				,					spaces + 2 )
    elif isinstance(snake_case__				,					torch.Tensor ):
        print(snake_case__				,					":"				,					val.size() )
    else:
        print(snake_case__				,					":"				,					snake_case__ )





def        lowerCAmelCase       (				snake_case__   :      Any				,					snake_case__   :      List[Any]				,					snake_case__   :      str				,					snake_case__   :      str )->			Union[str, Any]:
    # Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
    # for compatibility with later versions of NVIDIA Megatron-LM.
    # The inverse operation is performed inside Megatron-LM to read checkpoints:
    # https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
    # If param is the weight tensor of the self-attention block, the returned tensor
    # will have to be transposed one more time to be read by HuggingFace GPT2.
    A_								=					param.size()
    # other versions store [num_heads * num_splits * hidden_size, :]
    A_								=					(num_heads, num_splits, hidden_size) + input_shape[1:]
    A_								=					param.view(*snake_case__ )
    A_								=					param.transpose(0				,					1 ).contiguous()
    A_								=					param.view(*snake_case__ )
    return param





def        lowerCAmelCase       (				snake_case__   :      Dict				,					snake_case__   :      str )->			Optional[int]:
    A_								=					config.n_positions
    A_								=					config.n_layer
    A_								=					config.vocab_size
    A_								=					config.n_head
    A_								=					config.n_embd // config.n_head

    A_								=					sd_megatron["model.language_model.embedding.word_embeddings.weight"][:vocab_size, :]
    A_								=					{
        "transformer.wte.weight": word_embeddings,
        "transformer.wpe.weight": sd_megatron["model.language_model.embedding.position_embeddings.weight"],
        "transformer.ln_f.weight": sd_megatron["model.language_model.encoder.final_layernorm.weight"],
        "transformer.ln_f.bias": sd_megatron["model.language_model.encoder.final_layernorm.bias"],
    }

    A_								=					"model.language_model.encoder.layers."
    for i in range(snake_case__ ):
        A_								=					torch.tril(torch.ones((n_positions, n_positions)				,					dtype=torch.bool ) )
        A_								=					causal_mask.view(1				,					1				,					snake_case__				,					snake_case__ )
        A_								=					causal_mask
        A_								=					torch.tensor(-1e4				,					dtype=torch.bfloataa )

        A_								=					sd_megatron[f'{pf}{i}.input_layernorm.weight']
        A_								=					sd_megatron[f'{pf}{i}.input_layernorm.bias']

        A_								=					sd_megatron[f'{pf}{i}.self_attention.query_key_value.weight']
        A_								=					fix_query_key_value_ordering(snake_case__				,					3				,					snake_case__				,					snake_case__ )
        A_								=					vala.transpose(0				,					1 ).contiguous()

        A_								=					sd_megatron[f'{pf}{i}.self_attention.query_key_value.bias']
        A_								=					fix_query_key_value_ordering(snake_case__				,					3				,					snake_case__				,					snake_case__ )
        A_								=					vala

        A_								=					sd_megatron[f'{pf}{i}.self_attention.dense.weight'].transpose(
            0				,					1 )
        A_								=					sd_megatron[f'{pf}{i}.self_attention.dense.bias']
        A_								=					sd_megatron[f'{pf}{i}.post_attention_layernorm.weight']
        A_								=					sd_megatron[f'{pf}{i}.post_attention_layernorm.bias']
        A_								=					sd_megatron[f'{pf}{i}.mlp.dense_h_to_4h.weight'].transpose(0				,					1 )
        A_								=					sd_megatron[f'{pf}{i}.mlp.dense_h_to_4h.bias']
        A_								=					sd_megatron[f'{pf}{i}.mlp.dense_4h_to_h.weight'].transpose(
            0				,					1 )
        A_								=					sd_megatron[f'{pf}{i}.mlp.dense_4h_to_h.bias']

    # For LM head, transformers' wants the matrix to weight embeddings.
    A_								=					word_embeddings

    return sd_hf





def        lowerCAmelCase       (				snake_case__   :      Optional[int]				,					snake_case__   :      List[Any] )->			Dict:
    A_								=					64000
    A_								=					config_megatron["encoder_seq_length"]
    A_								=					config_megatron["hidden_size"]
    A_								=					config_megatron["num_layers"]
    A_								=					config_megatron["num_attention_heads"]
    A_								=					config_megatron["ffn_hidden_size"]
    A_								=					"gelu"
    A_								=					0.1
    A_								=					0.1
    A_								=					0.1
    A_								=					config_megatron["layernorm_epsilon"]  # 1e-5
    A_								=					config_megatron["init_method_std"]  # 0.02
    A_								=					config_megatron["apply_query_key_layer_scaling"]  # True
    A_								=					True
    A_								=					True

    # This identifies the 6.7B (7B) model which uses a different tokenizer
    if config_megatron["hidden_size"] == 4096:
        A_								=					1  # <|endoftext|>
        A_								=					1  # <|endoftext|>
        A_								=					0  # <unk>
    else:
        A_								=					2  # <s>
        A_								=					3  # <|endoftext|>
        A_								=					0  # <pad>

    return config_hf





def        lowerCAmelCase       (				snake_case__   :      Tuple )->			Dict:
    print(snake_case__ )

    A_								=					args.checkpoint_path
    A_								=					args.save_path
    if isfile(snake_case__ ):
        raise FileNotFoundError(f'ERROR! could not find file {checkpoint_path}' )

    # Load the model.
    A_								=					torch.load(snake_case__				,					map_location="cpu" )

    # Load the config.
    A_								=					checkpoint["hyper_parameters"]["cfg"]
    A_								=					GPTaConfig()
    A_								=					copy_config(config_hf=snake_case__				,					config_megatron=snake_case__ )
    A_								=					["GPT2LMHeadModel"]

    A_								=					checkpoint["state_dict"]

    # Convert.
    print("Converting" )
    A_								=					convert_megatron_checkpoint(snake_case__				,					snake_case__ )

    # Print the structure of converted state dict.
    if args.print_checkpoint_structure:
        recursive_print(snake_case__				,					snake_case__ )

    A_								=					"GPTSw3Tokenizer"

    # Store the config to file.
    print("Saving config" )
    config_hf.save_pretrained(snake_case__ )

    # Store the state_dict to file.
    A_								=					os.path.join(snake_case__				,					"pytorch_model.bin" )
    print(f'Saving checkpoint to "{output_checkpoint_file}"' )
    torch.save(snake_case__				,					snake_case__ )


if __name__ == "__main__":
     __magic_name__  :  Union[str, Any]     			=   argparse.ArgumentParser()
     parser.add_argument(
         '--checkpoint_path',
         type=str,
         required=True,
         help='e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000',
     )
     parser.add_argument('--save_path', type=str, required=True, help='e.g. /home/user/gpt-sw3/hf')
     parser.add_argument('--print-checkpoint-structure', action='store_true')
     __magic_name__  :  str     			=   parser.parse_args()
     main(_args)






