

"""simple docstring"""



import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
 import torch

SCREAMING_SNAKE_CASE_       :						Any					=    logging.get_logger(__name__)





def 			_snake_case  (     ):
     return importlib.util.find_spec("""deepspeed"""      ) is not None


if is_accelerate_available() and is_deepspeed_available():
 from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
 # Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
 # Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
 from builtins import object as DeepSpeedConfig



class        a    (							_lowerCamelCase					):



    """simple docstring"""



    def __init__(  self:			Tuple     ,    UpperCamelCase:			str							):






         """simple docstring"""


         set_hf_deepspeed_config(self							)
         dep_version_check("""accelerate"""							)
         dep_version_check("""deepspeed"""							)
         super().__init__(UpperCamelCase							)



class        a    (							_lowerCamelCase					):



    """simple docstring"""



    def __init__(  self:			Dict     ,    UpperCamelCase:			Optional[Any]							):






         """simple docstring"""


         super().__init__(UpperCamelCase							)
         A__				      =	None
         A__				      =	[]

    def 					UpperCamelCase						(  self:			int							):






         """simple docstring"""


         if self._dtype is None:
              raise ValueError("""trainer_config_process() wasn't called yet to tell dtype"""							)
         return self._dtype

    def 					UpperCamelCase						(  self:			str     ,    UpperCamelCase:			List[Any]							):






         """simple docstring"""


         A__				      =	self.get_value(UpperCamelCase							)
         if val is None:
              return False
         else:
              return val == "auto"

    def 					UpperCamelCase						(  self:			Optional[int]     ,    UpperCamelCase:			List[str]     ,    UpperCamelCase:			Tuple     ,    UpperCamelCase:			List[str]=None     ,    UpperCamelCase:			List[Any]=True							):






         """simple docstring"""


         A__      ,					A__				      =	self.find_config_node(UpperCamelCase							)
         if config is None:
              return

         if config.get(UpperCamelCase							) == "auto":
              A__				      =	hf_val
              return

         if not must_match:
              return

         A__				      =	config.get(UpperCamelCase							)
         if ds_val is not None and ds_val != hf_val:
              self.mismatches.append(f"""- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}"""							)

    UpperCAmelCase    			=       partialmethod(_lowerCamelCase,						must_match=_lowerCamelCase					)

    def 					UpperCamelCase						(  self:			Optional[int]     ,    UpperCamelCase:			List[Any]							):






         """simple docstring"""


         A__				      =	args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
         self.fill_match(
             """train_micro_batch_size_per_gpu"""     ,    args.per_device_train_batch_size     ,    """per_device_train_batch_size"""							)
         self.fill_match("""gradient_accumulation_steps"""     ,    args.gradient_accumulation_steps     ,    """gradient_accumulation_steps"""							)
         self.fill_match("""train_batch_size"""     ,    UpperCamelCase     ,    """train_batch_size (calculated)"""							)
         self.fill_match("""gradient_clipping"""     ,    args.max_grad_norm     ,    """max_grad_norm"""							)

         self.fill_match("""optimizer.params.lr"""     ,    args.learning_rate     ,    """learning_rate"""							)
         self.fill_match("""optimizer.params.betas"""     ,    [args.adam_betaa, args.adam_betaa]     ,    """adam_beta1+adam_beta2"""							)
         self.fill_match("""optimizer.params.eps"""     ,    args.adam_epsilon     ,    """adam_epsilon"""							)
         self.fill_match("""optimizer.params.weight_decay"""     ,    args.weight_decay     ,    """weight_decay"""							)

         self.fill_only("""scheduler.params.warmup_min_lr"""     ,    0							)  # not a trainer arg
         self.fill_match("""scheduler.params.warmup_max_lr"""     ,    args.learning_rate     ,    """learning_rate"""							)
         # total_num_steps - will get set in trainer_config_finalize

         # fp16
         if args.fpaa or args.fpaa_full_eval:
              A__				      =	"""apex""" if args.fpaa_backend == """apex""" else """amp"""
         else:
              A__				      =	None

         if args.save_on_each_node:
              # deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
              A__				      =	self.config.get("""checkpoint"""     ,    {}							)
              A__				      =	args.save_on_each_node

         # amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
         # any here unless the user did the work
         self.fill_match(
             """fp16.enabled"""     ,    ((args.fpaa or args.fpaa_full_eval) and fpaa_backend == """amp""")     ,    """fp16|fp16_full_eval+fp16_backend(amp)"""     ,    )

         # apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
         # ZeRO features
         self.fill_match("""amp.enabled"""     ,    fpaa_backend == """apex"""     ,    """fp16+fp16_backend(apex)"""							)
         self.fill_match("""amp.opt_level"""     ,    args.fpaa_opt_level     ,    """fp16_opt_level"""							)

         self.fill_match("""bf16.enabled"""     ,    (args.bfaa or args.bfaa_full_eval)     ,    """bf16|bf16_full_eval"""							)

         # deepspeed's default mode is fp16 unless there is a config that says differently
         if self.is_true("""bf16.enabled"""							):
              A__				      =	torch.bfloataa
         elif self.is_false("""fp16.enabled"""							):
              A__				      =	torch.floataa
         else:
              A__				      =	torch.floataa





    def 					UpperCamelCase						(  self:			Optional[int]     ,    UpperCamelCase:			Optional[int]     ,    UpperCamelCase:			Tuple     ,    UpperCamelCase:			Optional[int]							):






         """simple docstring"""


         A__				      =	[
             """zero_optimization.reduce_bucket_size""",
             """zero_optimization.stage3_prefetch_bucket_size""",
             """zero_optimization.stage3_param_persistence_threshold""",
         ]
         A__				      =	[x for x in hidden_size_based_keys if self.is_auto(UpperCamelCase							)]

         if len(UpperCamelCase							) > 0:
              if hasattr(model.config     ,    """hidden_size"""							):
                   A__				      =	model.config.hidden_size
              elif hasattr(model.config     ,    """hidden_sizes"""							):
                   # if there are many hidden sizes pick the largest one
                   A__				      =	max(model.config.hidden_sizes							)
              else:
                   raise ValueError(
                       """The model's config file has neither `hidden_size` nor `hidden_sizes` entry, """
                       """therefore it's not possible to automatically fill out the following `auto` entries """
                       f"""in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing """
                       """`auto` values for these keys with an integer value of your choice."""							)

              self.fill_only("""zero_optimization.reduce_bucket_size"""     ,    hidden_size * hidden_size							)
              if self.is_zeroa():
                   # automatically assign the optimal config values based on model config
                   self.fill_only("""zero_optimization.stage3_prefetch_bucket_size"""     ,    0.9 * hidden_size * hidden_size							)
                   self.fill_only("""zero_optimization.stage3_param_persistence_threshold"""     ,    10 * hidden_size							)

        # scheduler
         self.fill_match("""scheduler.params.total_num_steps"""     ,    UpperCamelCase     ,    """num_training_steps (calculated)"""							)
         self.fill_match("""scheduler.params.warmup_num_steps"""     ,    args.get_warmup_steps(UpperCamelCase							)     ,    """warmup_steps"""							)

         if len(self.mismatches							) > 0:
              A__				      =	"""\n""".join(self.mismatches							)
              raise ValueError(
                  """Please correct the following DeepSpeed config values that mismatch TrainingArguments"""
                  f""" values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to 'auto'."""							)

SCREAMING_SNAKE_CASE_       :						Tuple					=    None





def 			_snake_case  (     UpperCAmelCase_	:       Union[str, Any]      ):
     # this is a special weakref global object to allow us to get to Deepspeed config from APIs
     # that don't have an easy way to get to the Deepspeed config outside of the Trainer domain.
     global _hf_deepspeed_config_weak_ref
     # will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
     A__				      =	weakref.ref(UpperCAmelCase_      )





def 			_snake_case  (     ):
     # useful for unit tests to ensure the global state doesn't leak - call from `tearDown` method
     global _hf_deepspeed_config_weak_ref
     A__				      =	None





def 			_snake_case  (     ):
     if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
          return _hf_deepspeed_config_weak_ref().is_zeroa()
     else:
          return False





def 			_snake_case  (     ):
     if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
          return _hf_deepspeed_config_weak_ref().config
     else:
          return None





def 			_snake_case  (     UpperCAmelCase_	:       Dict       , UpperCAmelCase_	:       List[Any]       , UpperCAmelCase_	:       Optional[int]       , UpperCAmelCase_	:       Optional[Any]       , UpperCAmelCase_	:       Optional[int]      ):
     from accelerate.utils import DummyOptim, DummyScheduler

     A__				      =	hf_deepspeed_config.config

     # Optimizer + Scheduler
     # Currently supported combos:
     # 1. DS scheduler + DS optimizer: Yes
     # 2. HF scheduler + HF optimizer: Yes
     # 3. DS scheduler + HF optimizer: Yes
     # 4. HF scheduler + DS optimizer: No
     #
     # Unless Offload is enabled in which case it's:
     # 1. DS scheduler + DS optimizer: Yes
     # 2. HF scheduler + HF optimizer: Mostly*
     # 3. DS scheduler + HF optimizer: Mostly*
     # 4. HF scheduler + DS optimizer: No
     #
     # Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

     A__				      =	None
     if "optimizer" in config:
          if args.adafactor:
               raise ValueError(
                   """--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. """
                   """Only one optimizer can be configured."""      )
          A__				      =	DummyOptim(params=UpperCAmelCase_      )
     else:
          if hf_deepspeed_config.is_offload():
               logger.info(
                   """Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the"""
                   """ custom optimizer has both CPU and GPU implementation (except LAMB)"""      )

          # ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
          # But trainer uses AdamW by default.
          A__				      =	trainer.create_optimizer()
          # To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
          A__				      =	True

     A__				      =	None
     if "scheduler" in config:
          A__				      =	DummyScheduler(UpperCAmelCase_      )
     else:
          if isinstance(UpperCAmelCase_       , UpperCAmelCase_      ):
               raise ValueError(
                   """Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. """
                   """Please configure a scheduler in the DeepSpeed config."""      )
          A__				      =	trainer.create_scheduler(num_training_steps=UpperCAmelCase_       , optimizer=UpperCAmelCase_      )

     return optimizer, lr_scheduler





def 			_snake_case  (     UpperCAmelCase_	:       Tuple       , UpperCAmelCase_	:       int       , UpperCAmelCase_	:       Union[str, Any]=False      ):
     from deepspeed.utils import logger as ds_logger

     A__				      =	trainer.model
     A__				      =	trainer.args

     A__				      =	trainer.accelerator.state.deepspeed_plugin.hf_ds_config

     # resume config update - some bits like `model` and `num_training_steps` only become available during train
     hf_deepspeed_config.trainer_config_finalize(UpperCAmelCase_       , UpperCAmelCase_       , UpperCAmelCase_      )

     # set the Deepspeed log level consistent with the Trainer
     ds_logger.setLevel(args.get_process_log_level()      )

     if inference:
          # only Z3 makes sense for the inference
          if not hf_deepspeed_config.is_zeroa():
               raise ValueError("""ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config"""      )

          # in case the training config is re-used for inference
          hf_deepspeed_config.del_config_sub_tree("""optimizer"""      )
          hf_deepspeed_config.del_config_sub_tree("""lr_scheduler"""      )
          A__      ,					A__				      =	None, None
          A__				      =	None
     else:
          A__				      =	None  # important for when deepspeed_init is used as re-init
          A__				      =	list(filter(lambda UpperCAmelCase_      : p.requires_grad       , model.parameters()      )      )
          A__      ,					A__				      =	deepspeed_optim_sched(
              UpperCAmelCase_       , UpperCAmelCase_       , UpperCAmelCase_       , UpperCAmelCase_       , UpperCAmelCase_      )

     # keep for quick debug:
     # from pprint import pprint; pprint(config)

     return optimizer, lr_scheduler





def 			_snake_case  (     UpperCAmelCase_	:       Dict       , UpperCAmelCase_	:       str      ):
     # it's possible that the user is trying to resume from model_path, which doesn't necessarily
     # contain a deepspeed checkpoint. e.g. examples just check if the dir exists and assume it's
     # a resume from a checkpoint and not just a local pretrained weight. So we check here if the
     # path contains what looks like a deepspeed checkpoint
     import glob

     A__				      =	sorted(glob.glob(F"""{checkpoint_path}/global_step*"""      )      )

     if len(UpperCAmelCase_      ) > 0:
          logger.info(F"""Attempting to resume from {checkpoint_path}"""      )
          # this magically updates self.optimizer and self.lr_scheduler
          A__      ,					A__				      =	deepspeed_engine.load_checkpoint(
              UpperCAmelCase_       , load_optimizer_states=UpperCAmelCase_       , load_lr_scheduler_states=UpperCAmelCase_      )
          if load_path is None:
               raise ValueError(F"""[deepspeed] failed to resume from checkpoint {checkpoint_path}"""      )
     else:
          raise ValueError(F"""Can't find a valid checkpoint at {checkpoint_path}"""      )









"""simple docstring"""



from typing import List

from tokenizers import NormalizedString, PreTokenizedString, normalizers



class        a    :



    """simple docstring"""



    def __init__(  self:			List[Any]     ,    UpperCamelCase:			str							):






         """simple docstring"""


         A__				      =	vocab
         A__				      =	normalizers.BertNormalizer(
             clean_text=UpperCamelCase     ,    handle_chinese_chars=UpperCamelCase     ,    strip_accents=UpperCamelCase     ,    lowercase=UpperCamelCase     ,    )
         try:
              import rjieba
         except ImportError:
              raise ImportError(
                  """You need to install rjieba to use RoFormerTokenizer. """
                  """See https://pypi.org/project/rjieba/ for installation."""							)
         A__				      =	rjieba

    def 					UpperCamelCase						(  self:			Optional[Any]     ,    UpperCamelCase:			int     ,    UpperCamelCase:			NormalizedString							):






         """simple docstring"""


         A__				      =	[]

         # this code slice normalized_string is too slow (6s) but test_alignement_methods can pass
         for token, start, end in self.jieba.tokenize(str(UpperCamelCase							)     ,    hmm=UpperCamelCase							):
              if token in self.vocab:
                   splits.append(normalized_string[start:end]							)
              else:
                   A__				      =	self.normalizers.normalize_str(UpperCamelCase							).split()
                   for token in token_list:
                        if token:
                             A__				      =	start + len(UpperCamelCase							)
                             splits.append(normalized_string[start:end]							)
                             A__				      =	end

        # this code test_alignement_methods can't pass but fast (300ms)
        # for token in self.jieba.cut(str(normalized_string), False):
        #     if token in self.vocab:
        #         splits.append(NormalizedString(token))
        #     else:
        #         token_list = self.normalizers.normalize_str(token).split()
        #         for token in token_list:
        #             if token:
        #                 splits.append(NormalizedString(token))

         return splits





    def 					UpperCamelCase						(  self:			Tuple     ,    UpperCamelCase:			PreTokenizedString							):






         """simple docstring"""


         pretok.split(self.jieba_split							)






