




"""simple docstring"""






from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


lowercase__     						=						logging.get_logger(__name__)


lowercase__     						=						{
    'Salesforce/codegen-350M-nl': 'https://huggingface.co/Salesforce/codegen-350M-nl/resolve/main/config.json',
    'Salesforce/codegen-350M-multi': 'https://huggingface.co/Salesforce/codegen-350M-multi/resolve/main/config.json',
    'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/config.json',
    'Salesforce/codegen-2B-nl': 'https://huggingface.co/Salesforce/codegen-2B-nl/resolve/main/config.json',
    'Salesforce/codegen-2B-multi': 'https://huggingface.co/Salesforce/codegen-2B-multi/resolve/main/config.json',
    'Salesforce/codegen-2B-mono': 'https://huggingface.co/Salesforce/codegen-2B-mono/resolve/main/config.json',
    'Salesforce/codegen-6B-nl': 'https://huggingface.co/Salesforce/codegen-6B-nl/resolve/main/config.json',
    'Salesforce/codegen-6B-multi': 'https://huggingface.co/Salesforce/codegen-6B-multi/resolve/main/config.json',
    'Salesforce/codegen-6B-mono': 'https://huggingface.co/Salesforce/codegen-6B-mono/resolve/main/config.json',
    'Salesforce/codegen-16B-nl': 'https://huggingface.co/Salesforce/codegen-16B-nl/resolve/main/config.json',
    'Salesforce/codegen-16B-multi': 'https://huggingface.co/Salesforce/codegen-16B-multi/resolve/main/config.json',
    'Salesforce/codegen-16B-mono': 'https://huggingface.co/Salesforce/codegen-16B-mono/resolve/main/config.json',
}






class lowerCAmelCase__	(     lowerCAmelCase__						):






			'''simple docstring'''



			lowerCamelCase__      =			"""codegen"""
			lowerCamelCase__      =			{
			    """max_position_embeddings""": """n_positions""",
			    """hidden_size""": """n_embd""",
			    """num_attention_heads""": """n_head""",
			    """num_hidden_layers""": """n_layer""",
			}
			def __init__(  self  ,      lowercase=50400  ,      lowercase=2048  ,      lowercase=2048  ,      lowercase=4096  ,      lowercase=28  ,      lowercase=16  ,      lowercase=64  ,      lowercase=None  ,      lowercase="gelu_new"  ,      lowercase=0.0  ,      lowercase=0.0  ,      lowercase=0.0  ,      lowercase=1E-5  ,      lowercase=0.02  ,      lowercase=True  ,      lowercase=50256  ,      lowercase=50256  ,      lowercase=False  ,      **lowercase  ,      ):
							_lowerCamelCase     :			Union[str, Any]    =    vocab_size
							_lowerCamelCase     :			Optional[Any]    =    n_ctx
							_lowerCamelCase     :			Optional[int]    =    n_positions
							_lowerCamelCase     :			int    =    n_embd
							_lowerCamelCase     :			Optional[Any]    =    n_layer
							_lowerCamelCase     :			List[Any]    =    n_head
							_lowerCamelCase     :			Optional[int]    =    n_inner
							_lowerCamelCase     :			Optional[int]    =    rotary_dim
							_lowerCamelCase     :			int    =    activation_function
							_lowerCamelCase     :			List[str]    =    resid_pdrop
							_lowerCamelCase     :			Optional[int]    =    embd_pdrop
							_lowerCamelCase     :			Tuple    =    attn_pdrop
							_lowerCamelCase     :			int    =    layer_norm_epsilon
							_lowerCamelCase     :			Optional[Any]    =    initializer_range
							_lowerCamelCase     :			str    =    use_cache

							_lowerCamelCase     :			List[Any]    =    bos_token_id
							_lowerCamelCase     :			Tuple    =    eos_token_id

							super().__init__(
							    bos_token_id=UpperCamelCase__  ,      eos_token_id=UpperCamelCase__  ,      tie_word_embeddings=UpperCamelCase__  ,      **UpperCamelCase__  )






class lowerCAmelCase__	(     lowerCAmelCase__						):






			'''simple docstring'''



			def __init__(  self  ,      lowercase  ,      lowercase = "default"  ,      lowercase = None  ,      lowercase = False  ,      ):
							super().__init__(UpperCamelCase__  ,      task=UpperCamelCase__  ,      patching_specs=UpperCamelCase__  ,      use_past=UpperCamelCase__  )
							if not getattr(self._config  ,      'pad_token_id'  ,      UpperCamelCase__  ):
											# TODO: how to do that better?
											_lowerCamelCase     :			Union[str, Any]    =    0
			@property
			def 					A_    (  self  ):
							_lowerCamelCase     :			List[Any]    =    OrderedDict({'input_ids': {0: 'batch', 1: 'sequence'}}  )
							if self.use_past:
											self.fill_with_past_key_values_(UpperCamelCase__  ,      direction='inputs'  )
											_lowerCamelCase     :			Optional[Any]    =    {0: "batch", 1: "past_sequence + sequence"}
							else:
											_lowerCamelCase     :			Dict    =    {0: "batch", 1: "sequence"}

							return common_inputs
			@property
			def 					A_    (  self  ):
							return self._config.n_layer
			@property
			def 					A_    (  self  ):
							return self._config.n_head
			def 					A_    (  self  ,      lowercase  ,      lowercase = -1  ,      lowercase = -1  ,      lowercase = False  ,      lowercase = None  ,      ):
							_lowerCamelCase     :			int    =    super(UpperCamelCase__  ,      self  ).generate_dummy_inputs(
							    UpperCamelCase__  ,      batch_size=UpperCamelCase__  ,      seq_length=UpperCamelCase__  ,      is_pair=UpperCamelCase__  ,      framework=UpperCamelCase__  )

							# We need to order the input in the way they appears in the forward()
							_lowerCamelCase     :			int    =    OrderedDict({'input_ids': common_inputs['input_ids']}  )

							# Need to add the past_keys
							if self.use_past:
											if not is_torch_available():
															raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'  )
											else:
															import torch

															_lowerCamelCase     :			Optional[int]    =    common_inputs["input_ids"].shape
															# Not using the same length for past_key_values
															_lowerCamelCase     :			List[Any]    =    seqlen + 2
															_lowerCamelCase     :			Any    =    (
															    batch,
															    self.num_attention_heads,
															    past_key_values_length,
															    self._config.hidden_size // self.num_attention_heads,
															)
															_lowerCamelCase     :			Union[str, Any]    =    [
															    (torch.zeros(UpperCamelCase__  ), torch.zeros(UpperCamelCase__  )) for _ in range(self.num_layers  )
															]

							_lowerCamelCase     :			str    =    common_inputs["attention_mask"]
							if self.use_past:
											_lowerCamelCase     :			Optional[Any]    =    ordered_inputs["attention_mask"].dtype
											_lowerCamelCase     :			Union[str, Any]    =    torch.cat(
											    [ordered_inputs['attention_mask'], torch.ones(UpperCamelCase__  ,      UpperCamelCase__  ,      dtype=UpperCamelCase__  )]  ,      dim=1  )

							return ordered_inputs






			@property
			def 					A_    (  self  ):
							return 13





import argparse
from pathlib import Path

from transformers import AutoConfig, AutoTokenizer, RagConfig, RagSequenceForGeneration, RagTokenForGeneration

def      A      (      _SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE = None ,_SCREAMING_SNAKE_CASE = None ,_SCREAMING_SNAKE_CASE = None ,)	-> List[str]:
       if config_name_or_path is None:
              lowerCamelCase   :      Any			 =					"facebook/rag-token-base" if model_type == "rag_token" else "facebook/rag-sequence-base"

       if generator_tokenizer_name_or_path is None:
              lowerCamelCase   :      Dict			 =					generator_name_or_path

       if question_encoder_tokenizer_name_or_path is None:
              lowerCamelCase   :      Any			 =					question_encoder_name_or_path

       lowerCamelCase   :      str			 =					RagTokenForGeneration if model_type == "rag_token" else RagSequenceForGeneration

       # Save model.
       lowerCamelCase   :      List[Any]			 =					RagConfig.from_pretrained(_SCREAMING_SNAKE_CASE    )
       lowerCamelCase   :      Union[str, Any]			 =					AutoConfig.from_pretrained(_SCREAMING_SNAKE_CASE    )
       lowerCamelCase   :      Optional[int]			 =					AutoConfig.from_pretrained(_SCREAMING_SNAKE_CASE    )

       lowerCamelCase   :      Optional[Any]			 =					gen_config
       lowerCamelCase   :      Optional[Any]			 =					question_encoder_config

       lowerCamelCase   :      List[Any]			 =					model_class.from_pretrained_question_encoder_generator(
           _SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE ,config=_SCREAMING_SNAKE_CASE    )
       rag_model.save_pretrained(_SCREAMING_SNAKE_CASE    )

       # Sanity check.
       model_class.from_pretrained(_SCREAMING_SNAKE_CASE    )

       # Save tokenizers.
       lowerCamelCase   :      List[str]			 =					AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    )
       gen_tokenizer.save_pretrained(dest_dir / "generator_tokenizer/"    )
       lowerCamelCase   :      int			 =					AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    )
       question_encoder_tokenizer.save_pretrained(dest_dir / "question_encoder_tokenizer/"    )


if __name__ == "__main__":
       SCREAMING_SNAKE_CASE__       :      Any  =   argparse.ArgumentParser()
       parser.add_argument(
           '--model_type',
           choices=['rag_sequence', 'rag_token'],
           required=True,
           type=str,
           help='RAG model type: rag_sequence, rag_token',
       )
       parser.add_argument('--dest', type=str, required=True, help='Path to the output checkpoint directory.')
       parser.add_argument('--generator_name_or_path', type=str, required=True, help='Generator model identifier')
       parser.add_argument(
           '--question_encoder_name_or_path', type=str, required=True, help='Question encoder model identifier'
       )

       parser.add_argument(
           '--generator_tokenizer_name_or_path',
           type=str,
           help='Generator tokenizer identifier, if not specified, resolves to ``generator_name_or_path``',
       )
       parser.add_argument(
           '--question_encoder_tokenizer_name_or_path',
           type=str,
           help='Question encoder tokenizer identifier, if not specified, resolves to ``question_encoder_name_or_path``',
       )
       parser.add_argument(
           '--config_name_or_path',
           type=str,
           help=(
               'Identifier of the model config to use, if not provided, resolves to a base config for a given'
               ' ``model_type``'
           ),
       )

       SCREAMING_SNAKE_CASE__       :      Union[str, Any]  =   parser.parse_args()

       SCREAMING_SNAKE_CASE__       :      Optional[Any]  =   Path(args.dest)
       dest_dir.mkdir(exist_ok=True)

       consolidate(
           args.model_type,
           args.generator_name_or_path,
           args.question_encoder_name_or_path,
           dest_dir,
           args.config_name_or_path,
           args.generator_tokenizer_name_or_path,
           args.question_encoder_tokenizer_name_or_path,
       )






