


import re
from typing import Callable, List, Optional, Union

import tensorflow as tf


try:
    from tensorflow.keras.optimizers.legacy import Adam
except ImportError:
    from tensorflow.keras.optimizers import Adam







class 			_lowercase (							tf.keras.optimizers.schedules.LearningRateSchedule):


       """simple docstring"""




       def __init__(   self							:			List[str]					,      __lowerCamelCase							:			float					,      __lowerCamelCase							:			Callable					,      __lowerCamelCase							:			int					,      __lowerCamelCase							:			float = 1.0					,      __lowerCamelCase							:			str = None					,      ):


             '''simple docstring'''



             super().__init__()
             lowerCamelCase__		:				Dict    =  initial_learning_rate
             lowerCamelCase__		:				Any    =  warmup_steps
             lowerCamelCase__		:				int    =  power
             lowerCamelCase__		:				Any    =  decay_schedule_fn
             lowerCamelCase__		:				Union[str, Any]    =  name





       def __call__(   self							:			Union[str, Any]					,      __lowerCamelCase							:			List[str]       ):


             '''simple docstring'''



             with tf.name_scope(self.name or "WarmUp"       ) as name:
                   # Implements polynomial warmup. i.e., if global_step < warmup_steps, the
                   # learning rate will be `global_step/num_warmup_steps * init_lr`.
                   lowerCamelCase__		:				Any    =  tf.cast(__SCREAMING_SNAKE_CASE					,      tf.floataa       )
                   lowerCamelCase__		:				Tuple    =  tf.cast(self.warmup_steps					,      tf.floataa       )
                   lowerCamelCase__		:				Any    =  global_step_float / warmup_steps_float
                   lowerCamelCase__		:				Dict    =  self.initial_learning_rate * tf.math.pow(__SCREAMING_SNAKE_CASE					,      self.power       )
                   return tf.cond(
                       global_step_float < warmup_steps_float					,      lambda: warmup_learning_rate					,      lambda: self.decay_schedule_fn(step - self.warmup_steps       )					,      name=__SCREAMING_SNAKE_CASE					,      )





       def 						lowerCAmelCase     (   self							:			List[str]       ):


             '''simple docstring'''



             return {
                 "initial_learning_rate": self.initial_learning_rate,
                 "decay_schedule_fn": self.decay_schedule_fn,
                 "warmup_steps": self.warmup_steps,
                 "power": self.power,
                 "name": self.name,
             }



def       lowercase_		(		_A			:    int    ,					_A			:    List[Any]    ,					_A			:    Any    ,					_A			:    int = 0.0    ,					_A			:    Optional[int] = 0.9    ,					_A			:    Tuple = 0.999    ,					_A			:    Optional[int] = 1E-8    ,					_A			:    List[Any] = None    ,					_A			:    List[Any] = None    ,					_A			:    Tuple = 0.0    ,					_A			:    Any = 1.0    ,					_A			:    Union[str, Any] = None    ,					):

      """simple docstring"""




      lowerCamelCase__		:				Optional[Any]    =  tf.keras.optimizers.schedules.PolynomialDecay(
          initial_learning_rate=_UpperCAmelCase    ,					decay_steps=num_train_steps - num_warmup_steps    ,					end_learning_rate=init_lr * min_lr_ratio    ,					power=_UpperCAmelCase    ,					)
      if num_warmup_steps:
            lowerCamelCase__		:				Tuple    =  WarmUp(
                initial_learning_rate=_UpperCAmelCase    ,					decay_schedule_fn=_UpperCAmelCase    ,					warmup_steps=_UpperCAmelCase    ,					)
      if weight_decay_rate > 0.0:
            lowerCamelCase__		:				Dict    =  AdamWeightDecay(
                learning_rate=_UpperCAmelCase    ,					weight_decay_rate=_UpperCAmelCase    ,					beta_a=_UpperCAmelCase    ,					beta_a=_UpperCAmelCase    ,					epsilon=_UpperCAmelCase    ,					clipnorm=_UpperCAmelCase    ,					global_clipnorm=_UpperCAmelCase    ,					exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"]    ,					include_in_weight_decay=_UpperCAmelCase    ,					)
      else:
            lowerCamelCase__		:				Dict    =  tf.keras.optimizers.Adam(
                learning_rate=_UpperCAmelCase    ,					beta_a=_UpperCAmelCase    ,					beta_a=_UpperCAmelCase    ,					epsilon=_UpperCAmelCase    ,					clipnorm=_UpperCAmelCase    ,					global_clipnorm=_UpperCAmelCase    ,					)
      # We return the optimizer and the LR scheduler in order to better track the
      # evolution of the LR independently of the optimizer.
      return optimizer, lr_schedule







class 			_lowercase (							__UpperCAmelCase):


       """simple docstring"""




       def __init__(   self							:			str					,      __lowerCamelCase							:			Union[float, tf.keras.optimizers.schedules.LearningRateSchedule] = 0.0_0_1					,      __lowerCamelCase							:			float = 0.9					,      __lowerCamelCase							:			float = 0.9_9_9					,      __lowerCamelCase							:			float = 1E-7					,      __lowerCamelCase							:			bool = False					,      __lowerCamelCase							:			float = 0.0					,      __lowerCamelCase							:			Optional[List[str]] = None					,      __lowerCamelCase							:			Optional[List[str]] = None					,      __lowerCamelCase							:			str = "AdamWeightDecay"					,      **__lowerCamelCase							:			Tuple					,      ):


             '''simple docstring'''



             super().__init__(__SCREAMING_SNAKE_CASE					,      __SCREAMING_SNAKE_CASE					,      __SCREAMING_SNAKE_CASE					,      __SCREAMING_SNAKE_CASE					,      __SCREAMING_SNAKE_CASE					,      __SCREAMING_SNAKE_CASE					,      **__SCREAMING_SNAKE_CASE       )
             lowerCamelCase__		:				Dict    =  weight_decay_rate
             lowerCamelCase__		:				List[Any]    =  include_in_weight_decay
             lowerCamelCase__		:				List[Any]    =  exclude_from_weight_decay





       @classmethod
       def 						lowerCAmelCase     (   cls							:			Optional[Any]					,      __lowerCamelCase							:			Union[str, Any]       ):


             '''simple docstring'''



             lowerCamelCase__		:				List[Any]    =  {"WarmUp": WarmUp}
             return super(__SCREAMING_SNAKE_CASE					,      cls       ).from_config(__SCREAMING_SNAKE_CASE					,      custom_objects=__SCREAMING_SNAKE_CASE       )





       def 						lowerCAmelCase     (   self							:			Optional[Any]					,      __lowerCamelCase							:			List[str]					,      __lowerCamelCase							:			Optional[int]					,      __lowerCamelCase							:			str       ):


             '''simple docstring'''



             super(__SCREAMING_SNAKE_CASE					,      self       )._prepare_local(__SCREAMING_SNAKE_CASE					,      __SCREAMING_SNAKE_CASE					,      __SCREAMING_SNAKE_CASE       )
             lowerCamelCase__		:				Optional[int]    =  tf.constant(
                 self.weight_decay_rate					,      name="adam_weight_decay_rate"       )





       def 						lowerCAmelCase     (   self							:			int					,      __lowerCamelCase							:			Tuple					,      __lowerCamelCase							:			Tuple					,      __lowerCamelCase							:			Tuple       ):


             '''simple docstring'''



             lowerCamelCase__		:				Optional[Any]    =  self._do_use_weight_decay(var.name       )
             if do_decay:
                   return var.assign_sub(
                       learning_rate * var * apply_state[(var.device, var.dtype.base_dtype)]["weight_decay_rate"]					,      use_locking=self._use_locking					,      )
             return tf.no_op()





       def 						lowerCAmelCase     (   self							:			Optional[int]					,      __lowerCamelCase							:			Tuple					,      __lowerCamelCase							:			Optional[Any]=None					,      **__lowerCamelCase							:			Optional[Any]       ):


             '''simple docstring'''



             lowerCamelCase__    ,   lowerCamelCase__		:				str    =  list(zip(*__SCREAMING_SNAKE_CASE       )       )
             return super(__SCREAMING_SNAKE_CASE					,      self       ).apply_gradients(zip(__SCREAMING_SNAKE_CASE					,      __SCREAMING_SNAKE_CASE       )					,      name=__SCREAMING_SNAKE_CASE					,      **__SCREAMING_SNAKE_CASE       )





       def 						lowerCAmelCase     (   self							:			Union[str, Any]					,      __lowerCamelCase							:			int					,      __lowerCamelCase							:			str					,      __lowerCamelCase							:			List[str]       ):


             '''simple docstring'''



             if apply_state is None:
                   return self._decayed_lr_t[var_dtype], {}

             lowerCamelCase__		:				str    =  apply_state or {}
             lowerCamelCase__		:				Dict    =  apply_state.get((var_device, var_dtype)       )
             if coefficients is None:
                   lowerCamelCase__		:				Optional[Any]    =  self._fallback_apply_state(__SCREAMING_SNAKE_CASE					,      __SCREAMING_SNAKE_CASE       )
                   lowerCamelCase__		:				List[Any]    =  coefficients

             return coefficients["lr_t"], {"apply_state": apply_state}





       def 						lowerCAmelCase     (   self							:			Tuple					,      __lowerCamelCase							:			Any					,      __lowerCamelCase							:			Optional[Any]					,      __lowerCamelCase							:			Union[str, Any]=None       ):


             '''simple docstring'''



             lowerCamelCase__    ,   lowerCamelCase__		:				Optional[Any]    =  self._get_lr(var.device					,      var.dtype.base_dtype					,      __SCREAMING_SNAKE_CASE       )
             lowerCamelCase__		:				str    =  self._decay_weights_op(__SCREAMING_SNAKE_CASE					,      __SCREAMING_SNAKE_CASE					,      __SCREAMING_SNAKE_CASE       )
             with tf.control_dependencies([decay]       ):
                   return super(__SCREAMING_SNAKE_CASE					,      self       )._resource_apply_dense(__SCREAMING_SNAKE_CASE					,      __SCREAMING_SNAKE_CASE					,      **__SCREAMING_SNAKE_CASE       )





       def 						lowerCAmelCase     (   self							:			Union[str, Any]					,      __lowerCamelCase							:			str					,      __lowerCamelCase							:			Any					,      __lowerCamelCase							:			Optional[int]					,      __lowerCamelCase							:			Optional[Any]=None       ):


             '''simple docstring'''



             lowerCamelCase__    ,   lowerCamelCase__		:				Union[str, Any]    =  self._get_lr(var.device					,      var.dtype.base_dtype					,      __SCREAMING_SNAKE_CASE       )
             lowerCamelCase__		:				List[str]    =  self._decay_weights_op(__SCREAMING_SNAKE_CASE					,      __SCREAMING_SNAKE_CASE					,      __SCREAMING_SNAKE_CASE       )
             with tf.control_dependencies([decay]       ):
                   return super(__SCREAMING_SNAKE_CASE					,      self       )._resource_apply_sparse(__SCREAMING_SNAKE_CASE					,      __SCREAMING_SNAKE_CASE					,      __SCREAMING_SNAKE_CASE					,      **__SCREAMING_SNAKE_CASE       )





       def 						lowerCAmelCase     (   self							:			Tuple       ):


             '''simple docstring'''



             lowerCamelCase__		:				Optional[int]    =  super().get_config()
             config.update({"weight_decay_rate": self.weight_decay_rate}       )
             return config





       def 						lowerCAmelCase     (   self							:			Any					,      __lowerCamelCase							:			Dict       ):


             '''simple docstring'''



             if self.weight_decay_rate == 0:
                   return False

             if self._include_in_weight_decay:
                   for r in self._include_in_weight_decay:
                         if re.search(__SCREAMING_SNAKE_CASE					,      __SCREAMING_SNAKE_CASE       ) is not None:
                               return True

             if self._exclude_from_weight_decay:
                   for r in self._exclude_from_weight_decay:
                         if re.search(__SCREAMING_SNAKE_CASE					,      __SCREAMING_SNAKE_CASE       ) is not None:
                               return False
             return True







class 			_lowercase (							__UpperCAmelCase):


       """simple docstring"""




       def __init__(   self							:			Tuple       ):


             '''simple docstring'''



             lowerCamelCase__		:				str    =  []
             lowerCamelCase__		:				Optional[Any]    =  None





       @property
       def 						lowerCAmelCase     (   self							:			Dict       ):


             '''simple docstring'''



             if self._accum_steps is None:
                   lowerCamelCase__		:				Dict    =  tf.Variable(
                       tf.constant(0					,      dtype=tf.intaa       )					,      trainable=__SCREAMING_SNAKE_CASE					,      synchronization=tf.VariableSynchronization.ON_READ					,      aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA					,      )

             return self._accum_steps.value()





       @property
       def 						lowerCAmelCase     (   self							:			Any       ):


             '''simple docstring'''



             if not self._gradients:
                   raise ValueError("The accumulator should be called first to initialize the gradients"       )
             return [gradient.value() if gradient is not None else gradient for gradient in self._gradients]





       def __call__(   self							:			List[Any]					,      __lowerCamelCase							:			Union[str, Any]       ):


             '''simple docstring'''



             if not self._gradients:
                   lowerCamelCase__		:				List[Any]    =  self.step  # Create the step variable.
                   self._gradients.extend(
                       [
                           tf.Variable(
                               tf.zeros_like(__SCREAMING_SNAKE_CASE       )					,      trainable=__SCREAMING_SNAKE_CASE					,      synchronization=tf.VariableSynchronization.ON_READ					,      aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA					,      )
                           if gradient is not None
                           else gradient
                           for gradient in gradients
                       ]       )
             if len(__SCREAMING_SNAKE_CASE       ) != len(self._gradients       ):
                   raise ValueError(f"Expected {len(self._gradients       )} gradients, but got {len(__SCREAMING_SNAKE_CASE       )}"       )

             for accum_gradient, gradient in zip(self._gradients					,      __SCREAMING_SNAKE_CASE       ):
                   if accum_gradient is not None and gradient is not None:
                         accum_gradient.assign_add(__SCREAMING_SNAKE_CASE       )

             self._accum_steps.assign_add(1       )





       def 						lowerCAmelCase     (   self							:			List[str]       ):


             '''simple docstring'''



             if not self._gradients:
                   return
             self._accum_steps.assign(0       )
             for gradient in self._gradients:
                   if gradient is not None:
                         gradient.assign(tf.zeros_like(__SCREAMING_SNAKE_CASE       )       )







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


__snake_case						:str      		=     logging.get_logger(__name__)

__snake_case						:int      		=     {'''vocab_file''': '''vocab.txt'''}

__snake_case						:List[Any]      		=     {
    '''vocab_file''': {
        '''YituTech/conv-bert-base''': '''https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt''',
        '''YituTech/conv-bert-medium-small''': (
            '''https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt'''
        ),
        '''YituTech/conv-bert-small''': '''https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt''',
    }
}

__snake_case						:List[str]      		=     {
    '''YituTech/conv-bert-base''': 512,
    '''YituTech/conv-bert-medium-small''': 512,
    '''YituTech/conv-bert-small''': 512,
}


__snake_case						:Optional[int]      		=     {
    '''YituTech/conv-bert-base''': {'''do_lower_case''': True},
    '''YituTech/conv-bert-medium-small''': {'''do_lower_case''': True},
    '''YituTech/conv-bert-small''': {'''do_lower_case''': True},
}

class     _A       (	__UpperCAmelCase    ):
       UpperCamelCase__  :    Optional[int]											=						VOCAB_FILES_NAMES
       UpperCamelCase__  :    str											=						PRETRAINED_VOCAB_FILES_MAP
       UpperCamelCase__  :    Optional[Any]											=						PRETRAINED_INIT_CONFIGURATION
       UpperCamelCase__  :    Optional[int]											=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       UpperCamelCase__  :    int											=						ConvBertTokenizer



       def __init__(		self							:     Optional[Any]     ,		__SCREAMING_SNAKE_CASE							:     Optional[Any]=None     ,		__SCREAMING_SNAKE_CASE							:     Dict=None     ,		__SCREAMING_SNAKE_CASE							:     Tuple=True     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]="[UNK]"     ,		__SCREAMING_SNAKE_CASE							:     int="[SEP]"     ,		__SCREAMING_SNAKE_CASE							:     List[Any]="[PAD]"     ,		__SCREAMING_SNAKE_CASE							:     int="[CLS]"     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]="[MASK]"     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]=True     ,		__SCREAMING_SNAKE_CASE							:     Union[str, Any]=None     ,		**__SCREAMING_SNAKE_CASE							:     Dict     ,		):


           '''simple docstring'''

           super().__init__(
               __SCREAMING_SNAKE_CASE     ,		tokenizer_file=__SCREAMING_SNAKE_CASE     ,		do_lower_case=__SCREAMING_SNAKE_CASE     ,		unk_token=__SCREAMING_SNAKE_CASE     ,		sep_token=__SCREAMING_SNAKE_CASE     ,		pad_token=__SCREAMING_SNAKE_CASE     ,		cls_token=__SCREAMING_SNAKE_CASE     ,		mask_token=__SCREAMING_SNAKE_CASE     ,		tokenize_chinese_chars=__SCREAMING_SNAKE_CASE     ,		strip_accents=__SCREAMING_SNAKE_CASE     ,		**__SCREAMING_SNAKE_CASE     ,		)

           __a						       =				json.loads(self.backend_tokenizer.normalizer.__getstate__())
           if (
               normalizer_state.get('''lowercase'''     ,		__SCREAMING_SNAKE_CASE) != do_lower_case
               or normalizer_state.get('''strip_accents'''     ,		__SCREAMING_SNAKE_CASE) != strip_accents
               or normalizer_state.get('''handle_chinese_chars'''     ,		__SCREAMING_SNAKE_CASE) != tokenize_chinese_chars
           ):
               __a						       =				getattr(__SCREAMING_SNAKE_CASE     ,		normalizer_state.pop('''type'''))
               __a						       =				do_lower_case
               __a						       =				strip_accents
               __a						       =				tokenize_chinese_chars
               __a						       =				normalizer_class(**__SCREAMING_SNAKE_CASE)

           __a						       =				do_lower_case



       def 		_lowerCamelCase			(		self							:     int     ,		__SCREAMING_SNAKE_CASE							:     Dict     ,		__SCREAMING_SNAKE_CASE							:     Any=None):


           '''simple docstring'''

           __a						       =				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

           return output



       def 		_lowerCamelCase			(		self							:     str     ,		__SCREAMING_SNAKE_CASE							:     List[int]     ,		__SCREAMING_SNAKE_CASE							:     Optional[List[int]] = None):


           '''simple docstring'''

           __a						       =				[self.sep_token_id]
           __a						       =				[self.cls_token_id]
           if token_ids_a is None:
               return len(cls + token_ids_a + sep) * [0]
           return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]





       def 		_lowerCamelCase			(		self							:     Optional[Any]     ,		__SCREAMING_SNAKE_CASE							:     str     ,		__SCREAMING_SNAKE_CASE							:     Optional[str] = None):


           '''simple docstring'''

           __a						       =				self._tokenizer.model.save(__SCREAMING_SNAKE_CASE     ,		name=__SCREAMING_SNAKE_CASE)
           return tuple(__SCREAMING_SNAKE_CASE)





