






'''simple docstring'''






import argparse
import pathlib

import fairseq
import torch
from fairseq.models.roberta import RobertaModel as FairseqRobertaModel
from fairseq.modules import TransformerSentenceEncoderLayer
from packaging import version

from transformers import RobertaConfig, RobertaForMaskedLM, RobertaForSequenceClassification
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.utils import logging


if version.parse(fairseq.__version__) < version.parse('''0.9.0'''):
      raise Exception('''requires fairseq >= 0.9.0''')


logging.set_verbosity_info()
__A     						= logging.get_logger(__name__)

__A     						= '''Hello world! cécé herlolip'''




def 							_SCREAMING_SNAKE_CASE	(					A	:							str	,		A	:							str	,		A	:							bool	)     ->			Any:







   """simple docstring"""
   __snake_case					:   Any  				=			FairseqRobertaModel.from_pretrained(A	)
   roberta.eval()  # disable dropout
   __snake_case					:   Optional[int]  				=			roberta.model.encoder.sentence_encoder
   __snake_case					:   List[Any]  				=			RobertaConfig(
       vocab_size=roberta_sent_encoder.embed_tokens.num_embeddings	,		hidden_size=roberta.args.encoder_embed_dim	,		num_hidden_layers=roberta.args.encoder_layers	,		num_attention_heads=roberta.args.encoder_attention_heads	,		intermediate_size=roberta.args.encoder_ffn_embed_dim	,		max_position_embeddings=5_14	,		type_vocab_size=1	,		layer_norm_eps=1e-5	,		)
   if classification_head:
      __snake_case					:   Dict  				=			roberta.model.classification_heads['mnli'].out_proj.weight.shape[0]
   print('Our BERT config:'	,		A	)

   __snake_case					:   List[Any]  				=			RobertaForSequenceClassification(A	) if classification_head else RobertaForMaskedLM(A	)
   model.eval()

   # Now let's copy all the weights.
   # Embeddings
   __snake_case					:   List[str]  				=			roberta_sent_encoder.embed_tokens.weight
   __snake_case					:   Optional[int]  				=			roberta_sent_encoder.embed_positions.weight
   __snake_case					:   str  				=			torch.zeros_like(
       model.roberta.embeddings.token_type_embeddings.weight	)  # just zero them out b/c RoBERTa doesn't use them.
   __snake_case					:   Union[str, Any]  				=			roberta_sent_encoder.emb_layer_norm.weight
   __snake_case					:   str  				=			roberta_sent_encoder.emb_layer_norm.bias

   for i in range(config.num_hidden_layers	):
      # Encoder: start of layer
      __snake_case					:   BertLayer  				=			model.roberta.encoder.layer[i]
      __snake_case					:   TransformerSentenceEncoderLayer  				=			roberta_sent_encoder.layers[i]

      # self attention
      __snake_case					:   BertSelfAttention  				=			layer.attention.self
      assert (
          roberta_layer.self_attn.k_proj.weight.data.shape
          == roberta_layer.self_attn.q_proj.weight.data.shape
          == roberta_layer.self_attn.v_proj.weight.data.shape
          == torch.Size((config.hidden_size, config.hidden_size)	)
      )

      __snake_case					:   Any  				=			roberta_layer.self_attn.q_proj.weight
      __snake_case					:   int  				=			roberta_layer.self_attn.q_proj.bias
      __snake_case					:   str  				=			roberta_layer.self_attn.k_proj.weight
      __snake_case					:   Optional[Any]  				=			roberta_layer.self_attn.k_proj.bias
      __snake_case					:   Optional[Any]  				=			roberta_layer.self_attn.v_proj.weight
      __snake_case					:   Any  				=			roberta_layer.self_attn.v_proj.bias

      # self-attention output
      __snake_case					:   BertSelfOutput  				=			layer.attention.output
      assert self_output.dense.weight.shape == roberta_layer.self_attn.out_proj.weight.shape
      __snake_case					:   Any  				=			roberta_layer.self_attn.out_proj.weight
      __snake_case					:   int  				=			roberta_layer.self_attn.out_proj.bias
      __snake_case					:   List[Any]  				=			roberta_layer.self_attn_layer_norm.weight
      __snake_case					:   Optional[int]  				=			roberta_layer.self_attn_layer_norm.bias

      # intermediate
      __snake_case					:   BertIntermediate  				=			layer.intermediate
      assert intermediate.dense.weight.shape == roberta_layer.fca.weight.shape
      __snake_case					:   List[str]  				=			roberta_layer.fca.weight
      __snake_case					:   Tuple  				=			roberta_layer.fca.bias

      # output
      __snake_case					:   BertOutput  				=			layer.output
      assert bert_output.dense.weight.shape == roberta_layer.fca.weight.shape
      __snake_case					:   Any  				=			roberta_layer.fca.weight
      __snake_case					:   int  				=			roberta_layer.fca.bias
      __snake_case					:   Optional[Any]  				=			roberta_layer.final_layer_norm.weight
      __snake_case					:   Optional[Any]  				=			roberta_layer.final_layer_norm.bias
      # end of layer

   if classification_head:
      __snake_case					:   str  				=			roberta.model.classification_heads['mnli'].dense.weight
      __snake_case					:   List[Any]  				=			roberta.model.classification_heads['mnli'].dense.bias
      __snake_case					:   List[Any]  				=			roberta.model.classification_heads['mnli'].out_proj.weight
      __snake_case					:   List[str]  				=			roberta.model.classification_heads['mnli'].out_proj.bias
   else:
      # LM Head
      __snake_case					:   Optional[int]  				=			roberta.model.encoder.lm_head.dense.weight
      __snake_case					:   Tuple  				=			roberta.model.encoder.lm_head.dense.bias
      __snake_case					:   str  				=			roberta.model.encoder.lm_head.layer_norm.weight
      __snake_case					:   str  				=			roberta.model.encoder.lm_head.layer_norm.bias
      __snake_case					:   Union[str, Any]  				=			roberta.model.encoder.lm_head.weight
      __snake_case					:   Optional[int]  				=			roberta.model.encoder.lm_head.bias

   # Let's check that we get the same results.
   __snake_case					:   torch.Tensor  				=			roberta.encode(A	).unsqueeze(0	)  # batch of size 1

   __snake_case					:   Any  				=			model(A	)[0]
   if classification_head:
      __snake_case					:   str  				=			roberta.model.classification_heads['mnli'](roberta.extract_features(A	)	)
   else:
      __snake_case					:   Union[str, Any]  				=			roberta.model(A	)[0]
   print(our_output.shape	,		their_output.shape	)
   __snake_case					:   int  				=			torch.max(torch.abs(our_output - their_output	)	).item()
   print(F"""max_absolute_diff = {max_absolute_diff}"""	)  # ~ 1e-7
   __snake_case					:   Dict  				=			torch.allclose(A	,		A	,		atol=1e-3	)
   print('Do both models output the same tensors?'	,		'🔥' if success else '💩'	)
   if not success:
      raise Exception('Something went wRoNg'	)

   pathlib.Path(A	).mkdir(parents=A	,		exist_ok=A	)
   print(F"""Saving model to {pytorch_dump_folder_path}"""	)
   model.save_pretrained(A	)


if __name__ == "__main__":
      __A     						= argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          '''--roberta_checkpoint_path''', default=None, type=str, required=True, help='''Path the official PyTorch dump.'''
      )
      parser.add_argument(
          '''--pytorch_dump_folder_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
      )
      parser.add_argument(
          '''--classification_head''', action='''store_true''', help='''Whether to convert a final classification head.'''
      )
      __A     						= parser.parse_args()
      convert_roberta_checkpoint_to_pytorch(
          args.roberta_checkpoint_path, args.pytorch_dump_folder_path, args.classification_head
      )







'''simple docstring'''






import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
      import torch

__A     						= logging.get_logger(__name__)




def 							_SCREAMING_SNAKE_CASE	(					)     ->			Optional[Any]:







   """simple docstring"""
   return importlib.util.find_spec('deepspeed'	) is not None


if is_accelerate_available() and is_deepspeed_available():
      from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
      # Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
      # Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
      from builtins import object as DeepSpeedConfig




class a_     (     UpperCamelCase_				):







   def __init__(self       , __a)      ->   Optional[Any]:
      """simple docstring"""






      set_hf_deepspeed_config(self)
      dep_version_check('accelerate')
      dep_version_check('deepspeed')
      super().__init__(__a)




class a_     (     UpperCamelCase_				):







   def __init__(self       , __a)      ->   List[str]:
      """simple docstring"""






      super().__init__(__a)
      __snake_case					:   Dict  				=			None
      __snake_case					:   Optional[Any]  				=			[]







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Dict:
      """simple docstring"""






      if self._dtype is None:
         raise ValueError('trainer_config_process() wasn\'t called yet to tell dtype')
      return self._dtype







   def      SCREAMING_SNAKE_CASE__       (self       , __a)      ->   int:
      """simple docstring"""






      __snake_case					:   Dict  				=			self.get_value(__a)
      if val is None:
         return False
      else:
         return val == "auto"







   def      SCREAMING_SNAKE_CASE__       (self       , __a       , __a       , __a=None       , __a=True)      ->   int:
      """simple docstring"""






      __snake_case				,__snake_case					:   str  				=			self.find_config_node(__a)
      if config is None:
         return

      if config.get(__a) == "auto":
         __snake_case					:   List[Any]  				=			hf_val
         return

      if not must_match:
         return

      __snake_case					:   Optional[Any]  				=			config.get(__a)
      if ds_val is not None and ds_val != hf_val:
         self.mismatches.append(F"""- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}""")

   _snake_case	   =				partialmethod(UpperCamelCase_		,   must_match=UpperCamelCase_				)







   def      SCREAMING_SNAKE_CASE__       (self       , __a)      ->   Union[str, Any]:
      """simple docstring"""






      __snake_case					:   str  				=			args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
      self.fill_match(
          'train_micro_batch_size_per_gpu'       , args.per_device_train_batch_size       , 'per_device_train_batch_size')
      self.fill_match('gradient_accumulation_steps'       , args.gradient_accumulation_steps       , 'gradient_accumulation_steps')
      self.fill_match('train_batch_size'       , __a       , 'train_batch_size (calculated)')
      self.fill_match('gradient_clipping'       , args.max_grad_norm       , 'max_grad_norm')

      self.fill_match('optimizer.params.lr'       , args.learning_rate       , 'learning_rate')
      self.fill_match('optimizer.params.betas'       , [args.adam_betaa, args.adam_betaa]       , 'adam_beta1+adam_beta2')
      self.fill_match('optimizer.params.eps'       , args.adam_epsilon       , 'adam_epsilon')
      self.fill_match('optimizer.params.weight_decay'       , args.weight_decay       , 'weight_decay')

      self.fill_only('scheduler.params.warmup_min_lr'       , 0)  # not a trainer arg
      self.fill_match('scheduler.params.warmup_max_lr'       , args.learning_rate       , 'learning_rate')
      # total_num_steps - will get set in trainer_config_finalize

      # fp16
      if args.fpaa or args.fpaa_full_eval:
         __snake_case					:   Optional[Any]  				=			'apex' if args.fpaa_backend == 'apex' else 'amp'
      else:
         __snake_case					:   Optional[int]  				=			None

      if args.save_on_each_node:
         # deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
         __snake_case					:   Union[str, Any]  				=			self.config.get('checkpoint'       , {})
         __snake_case					:   List[str]  				=			args.save_on_each_node

      # amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
      # any here unless the user did the work
      self.fill_match(
          'fp16.enabled'       , ((args.fpaa or args.fpaa_full_eval) and fpaa_backend == 'amp')       , 'fp16|fp16_full_eval+fp16_backend(amp)'       , )

      # apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
      # ZeRO features
      self.fill_match('amp.enabled'       , fpaa_backend == 'apex'       , 'fp16+fp16_backend(apex)')
      self.fill_match('amp.opt_level'       , args.fpaa_opt_level       , 'fp16_opt_level')

      self.fill_match('bf16.enabled'       , (args.bfaa or args.bfaa_full_eval)       , 'bf16|bf16_full_eval')

      # deepspeed's default mode is fp16 unless there is a config that says differently
      if self.is_true('bf16.enabled'):
         __snake_case					:   List[str]  				=			torch.bfloataa
      elif self.is_false('fp16.enabled'):
         __snake_case					:   Union[str, Any]  				=			torch.floataa
      else:
         __snake_case					:   List[str]  				=			torch.floataa







   def      SCREAMING_SNAKE_CASE__       (self       , __a       , __a       , __a)      ->   Any:
      """simple docstring"""






      __snake_case					:   Union[str, Any]  				=			[
          'zero_optimization.reduce_bucket_size',
          'zero_optimization.stage3_prefetch_bucket_size',
          'zero_optimization.stage3_param_persistence_threshold',
      ]
      __snake_case					:   int  				=			[x for x in hidden_size_based_keys if self.is_auto(__a)]

      if len(__a) > 0:
         if hasattr(model.config       , 'hidden_size'):
            __snake_case					:   Optional[int]  				=			model.config.hidden_size
         elif hasattr(model.config       , 'hidden_sizes'):
            # if there are many hidden sizes pick the largest one
            __snake_case					:   Tuple  				=			max(model.config.hidden_sizes)
         else:
            raise ValueError(
                'The model\'s config file has neither `hidden_size` nor `hidden_sizes` entry, '
                'therefore it\'s not possible to automatically fill out the following `auto` entries '
                F"""in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing """
                '`auto` values for these keys with an integer value of your choice.')

         self.fill_only('zero_optimization.reduce_bucket_size'       , hidden_size * hidden_size)
         if self.is_zeroa():
            # automatically assign the optimal config values based on model config
            self.fill_only('zero_optimization.stage3_prefetch_bucket_size'       , 0.9 * hidden_size * hidden_size)
            self.fill_only('zero_optimization.stage3_param_persistence_threshold'       , 1_0 * hidden_size)

        # scheduler
      self.fill_match('scheduler.params.total_num_steps'       , __a       , 'num_training_steps (calculated)')
      self.fill_match('scheduler.params.warmup_num_steps'       , args.get_warmup_steps(__a)       , 'warmup_steps')

      if len(self.mismatches) > 0:
         __snake_case					:   Any  				=			'\n'.join(self.mismatches)
         raise ValueError(
             'Please correct the following DeepSpeed config values that mismatch TrainingArguments'
             F""" values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to 'auto'.""")


__A     						= None




def 							_SCREAMING_SNAKE_CASE	(					A	:							Dict	)     ->			Union[str, Any]:







   """simple docstring"""
   # this is a special weakref global object to allow us to get to Deepspeed config from APIs
   # that don't have an easy way to get to the Deepspeed config outside of the Trainer domain.
   global _hf_deepspeed_config_weak_ref
   # will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
   __snake_case					:   List[Any]  				=			weakref.ref(A	)




def 							_SCREAMING_SNAKE_CASE	(					)     ->			Union[str, Any]:







   """simple docstring"""
   # useful for unit tests to ensure the global state doesn't leak - call from `tearDown` method
   global _hf_deepspeed_config_weak_ref
   __snake_case					:   Optional[Any]  				=			None




def 							_SCREAMING_SNAKE_CASE	(					)     ->			str:







   """simple docstring"""
   if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
      return _hf_deepspeed_config_weak_ref().is_zeroa()
   else:
      return False




def 							_SCREAMING_SNAKE_CASE	(					)     ->			List[Any]:







   """simple docstring"""
   if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
      return _hf_deepspeed_config_weak_ref().config
   else:
      return None




def 							_SCREAMING_SNAKE_CASE	(					A	:							Dict	,		A	:							int	,		A	:							Optional[Any]	,		A	:							Optional[Any]	,		A	:							str	)     ->			Union[str, Any]:







   """simple docstring"""
   from accelerate.utils import DummyOptim, DummyScheduler

   __snake_case					:   Tuple  				=			hf_deepspeed_config.config

   # Optimizer + Scheduler
   # Currently supported combos:
   # 1. DS scheduler + DS optimizer: Yes
   # 2. HF scheduler + HF optimizer: Yes
   # 3. DS scheduler + HF optimizer: Yes
   # 4. HF scheduler + DS optimizer: No
   #
   # Unless Offload is enabled in which case it's:
   # 1. DS scheduler + DS optimizer: Yes
   # 2. HF scheduler + HF optimizer: Mostly*
   # 3. DS scheduler + HF optimizer: Mostly*
   # 4. HF scheduler + DS optimizer: No
   #
   # Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

   __snake_case					:   Any  				=			None
   if "optimizer" in config:
      if args.adafactor:
         raise ValueError(
             '--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. '
             'Only one optimizer can be configured.'	)
      __snake_case					:   Union[str, Any]  				=			DummyOptim(params=A	)
   else:
      if hf_deepspeed_config.is_offload():
         logger.info(
             'Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the'
             ' custom optimizer has both CPU and GPU implementation (except LAMB)'	)

      # ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
      # But trainer uses AdamW by default.
      __snake_case					:   Optional[int]  				=			trainer.create_optimizer()
      # To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
      __snake_case					:   Dict  				=			True

   __snake_case					:   List[Any]  				=			None
   if "scheduler" in config:
      __snake_case					:   Optional[int]  				=			DummyScheduler(A	)
   else:
      if isinstance(A	,		A	):
         raise ValueError(
             'Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. '
             'Please configure a scheduler in the DeepSpeed config.'	)
      __snake_case					:   int  				=			trainer.create_scheduler(num_training_steps=A	,		optimizer=A	)

   return optimizer, lr_scheduler




def 							_SCREAMING_SNAKE_CASE	(					A	:							Dict	,		A	:							List[Any]	,		A	:							str=False	)     ->			Union[str, Any]:







   """simple docstring"""
   from deepspeed.utils import logger as ds_logger

   __snake_case					:   int  				=			trainer.model
   __snake_case					:   Any  				=			trainer.args

   __snake_case					:   List[str]  				=			trainer.accelerator.state.deepspeed_plugin.hf_ds_config

   # resume config update - some bits like `model` and `num_training_steps` only become available during train
   hf_deepspeed_config.trainer_config_finalize(A	,		A	,		A	)

   # set the Deepspeed log level consistent with the Trainer
   ds_logger.setLevel(args.get_process_log_level()	)

   if inference:
      # only Z3 makes sense for the inference
      if not hf_deepspeed_config.is_zeroa():
         raise ValueError('ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config'	)

      # in case the training config is re-used for inference
      hf_deepspeed_config.del_config_sub_tree('optimizer'	)
      hf_deepspeed_config.del_config_sub_tree('lr_scheduler'	)
      __snake_case				,__snake_case					:   Any  				=			None, None
      __snake_case					:   int  				=			None
   else:
      __snake_case					:   str  				=			None  # important for when deepspeed_init is used as re-init
      __snake_case					:   Optional[int]  				=			list(filter(lambda A	: p.requires_grad	,		model.parameters()	)	)
      __snake_case				,__snake_case					:   List[Any]  				=			deepspeed_optim_sched(
          A	,		A	,		A	,		A	,		A	)

   # keep for quick debug:
   # from pprint import pprint; pprint(config)

   return optimizer, lr_scheduler





def 							_SCREAMING_SNAKE_CASE	(					A	:							Any	,		A	:							int	)     ->			List[Any]:







   """simple docstring"""
   # it's possible that the user is trying to resume from model_path, which doesn't necessarily
   # contain a deepspeed checkpoint. e.g. examples just check if the dir exists and assume it's
   # a resume from a checkpoint and not just a local pretrained weight. So we check here if the
   # path contains what looks like a deepspeed checkpoint
   import glob

   __snake_case					:   int  				=			sorted(glob.glob(F"""{checkpoint_path}/global_step*"""	)	)

   if len(A	) > 0:
      logger.info(F"""Attempting to resume from {checkpoint_path}"""	)
      # this magically updates self.optimizer and self.lr_scheduler
      __snake_case				,__snake_case					:   Optional[Any]  				=			deepspeed_engine.load_checkpoint(
          A	,		load_optimizer_states=A	,		load_lr_scheduler_states=A	)
      if load_path is None:
         raise ValueError(F"""[deepspeed] failed to resume from checkpoint {checkpoint_path}"""	)
   else:
      raise ValueError(F"""Can't find a valid checkpoint at {checkpoint_path}"""	)