



'''simple docstring'''





import shutil
import tempfile
import unittest
from unittest.mock import patch

from transformers import (
    DefaultFlowCallback,
    IntervalStrategy,
    PrinterCallback,
    ProgressCallback,
    Trainer,
    TrainerCallback,
    TrainingArguments,
    is_torch_available,
)
from transformers.testing_utils import require_torch


if is_torch_available():
 from transformers.trainer import DEFAULT_CALLBACKS

 from .test_trainer import RegressionDataset, RegressionModelConfig, RegressionPreTrainedModel






class        lowerCAmelCase_(				SCREAMING_SNAKE_CASE_     ):
  '''simple docstring'''




  def __init__(    self      )    ->		Union[str, Any]:
      lowerCAmelCase__ :  Union[str, Any]							      =					[]




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,**__UpperCAmelCase      )    ->		Optional[int]:
      self.events.append("""on_init_end"""      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,**__UpperCAmelCase      )    ->		int:
      self.events.append("""on_train_begin"""      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,**__UpperCAmelCase      )    ->		Optional[Any]:
      self.events.append("""on_train_end"""      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,**__UpperCAmelCase      )    ->		Dict:
      self.events.append("""on_epoch_begin"""      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,**__UpperCAmelCase      )    ->		Dict:
      self.events.append("""on_epoch_end"""      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,**__UpperCAmelCase      )    ->		Optional[int]:
      self.events.append("""on_step_begin"""      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,**__UpperCAmelCase      )    ->		Union[str, Any]:
      self.events.append("""on_step_end"""      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,**__UpperCAmelCase      )    ->		Optional[int]:
      self.events.append("""on_evaluate"""      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,**__UpperCAmelCase      )    ->		int:
      self.events.append("""on_predict"""      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,**__UpperCAmelCase      )    ->		Any:
      self.events.append("""on_save"""      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,**__UpperCAmelCase      )    ->		Any:
      self.events.append("""on_log"""      )






  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,**__UpperCAmelCase      )    ->		Optional[int]:
      self.events.append("""on_prediction_step"""      )







@require_torch
class        lowerCAmelCase_(				unittest.TestCase     ):
  '''simple docstring'''




  def 							UpperCAmelCase_	(    self      )    ->		Dict:
      lowerCAmelCase__ :  int							      =					tempfile.mkdtemp()




  def 							UpperCAmelCase_	(    self      )    ->		Tuple:
      shutil.rmtree(self.output_dir      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase=0  ,__UpperCAmelCase=0  ,__UpperCAmelCase=64  ,__UpperCAmelCase=64  ,__UpperCAmelCase=None  ,__UpperCAmelCase=False  ,**__UpperCAmelCase      )    ->		Optional[int]:
      # disable_tqdm in TrainingArguments has a flaky default since it depends on the level of logging. We make sure
      # its set to False since the tests later on depend on its value.
      lowerCAmelCase__ :  Tuple							      =					RegressionDataset(length=__UpperCAmelCase      )
      lowerCAmelCase__ :  Tuple							      =					RegressionDataset(length=__UpperCAmelCase      )
      lowerCAmelCase__ :  Union[str, Any]							      =					RegressionModelConfig(a=__UpperCAmelCase  ,b=__UpperCAmelCase      )
      lowerCAmelCase__ :  Union[str, Any]							      =					RegressionPreTrainedModel(__UpperCAmelCase      )

      lowerCAmelCase__ :  int							      =					TrainingArguments(self.output_dir  ,disable_tqdm=__UpperCAmelCase  ,report_to=[]  ,**__UpperCAmelCase      )
      return Trainer(
          __UpperCAmelCase  ,__UpperCAmelCase  ,train_dataset=__UpperCAmelCase  ,eval_dataset=__UpperCAmelCase  ,callbacks=__UpperCAmelCase  ,)




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase      )    ->		Dict:
      self.assertEqual(len(__UpperCAmelCase      )  ,len(__UpperCAmelCase      )      )

      # Order doesn't matter
      lowerCAmelCase__ :  Optional[int]							      =					sorted(__UpperCAmelCase  ,key=lambda __UpperCAmelCase      : cb.__name__ if isinstance(__UpperCAmelCase  ,__UpperCAmelCase      ) else cb.__class__.__name__      )
      lowerCAmelCase__ :  Optional[int]							      =					sorted(__UpperCAmelCase  ,key=lambda __UpperCAmelCase      : cb.__name__ if isinstance(__UpperCAmelCase  ,__UpperCAmelCase      ) else cb.__class__.__name__      )

      for cba, cba in zip(__UpperCAmelCase  ,__UpperCAmelCase      ):
          if isinstance(__UpperCAmelCase  ,__UpperCAmelCase      ) and isinstance(__UpperCAmelCase  ,__UpperCAmelCase      ):
              self.assertEqual(__UpperCAmelCase  ,__UpperCAmelCase      )
          elif isinstance(__UpperCAmelCase  ,__UpperCAmelCase      ) and not isinstance(__UpperCAmelCase  ,__UpperCAmelCase      ):
              self.assertEqual(__UpperCAmelCase  ,cba.__class__      )
          elif not isinstance(__UpperCAmelCase  ,__UpperCAmelCase      ) and isinstance(__UpperCAmelCase  ,__UpperCAmelCase      ):
              self.assertEqual(cba.__class__  ,__UpperCAmelCase      )
          else:
              self.assertEqual(__UpperCAmelCase  ,__UpperCAmelCase      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase      )    ->		Union[str, Any]:
      lowerCAmelCase__ :  Any							      =					["""on_init_end""", """on_train_begin"""]
      lowerCAmelCase__ :  Union[str, Any]							      =					0
      lowerCAmelCase__ :  Tuple							      =					len(trainer.get_eval_dataloader()      )
      lowerCAmelCase__ :  Optional[int]							      =					["""on_prediction_step"""] * len(trainer.get_eval_dataloader()      ) + ["""on_log""", """on_evaluate"""]
      for _ in range(trainer.state.num_train_epochs      ):
          expected_events.append("""on_epoch_begin"""      )
          for _ in range(__UpperCAmelCase      ):
              step += 1
              expected_events += ["on_step_begin", "on_step_end"]
              if step % trainer.args.logging_steps == 0:
                  expected_events.append("""on_log"""      )
              if trainer.args.evaluation_strategy == IntervalStrategy.STEPS and step % trainer.args.eval_steps == 0:
                  expected_events += evaluation_events.copy()
              if step % trainer.args.save_steps == 0:
                  expected_events.append("""on_save"""      )
          expected_events.append("""on_epoch_end"""      )
          if trainer.args.evaluation_strategy == IntervalStrategy.EPOCH:
              expected_events += evaluation_events.copy()
      expected_events += ["on_log", "on_train_end"]
      return expected_events




  def 							UpperCAmelCase_	(    self      )    ->		Dict:
      lowerCAmelCase__ :  Any							      =					self.get_trainer()
      lowerCAmelCase__ :  Tuple							      =					DEFAULT_CALLBACKS.copy() + [ProgressCallback]
      self.check_callbacks_equality(trainer.callback_handler.callbacks  ,__UpperCAmelCase      )

      # Callbacks passed at init are added to the default callbacks
      lowerCAmelCase__ :  Optional[Any]							      =					self.get_trainer(callbacks=[MyTestTrainerCallback]      )
      expected_callbacks.append(__UpperCAmelCase      )
      self.check_callbacks_equality(trainer.callback_handler.callbacks  ,__UpperCAmelCase      )

      # TrainingArguments.disable_tqdm controls if use ProgressCallback or PrinterCallback
      lowerCAmelCase__ :  Tuple							      =					self.get_trainer(disable_tqdm=__UpperCAmelCase      )
      lowerCAmelCase__ :  int							      =					DEFAULT_CALLBACKS.copy() + [PrinterCallback]
      self.check_callbacks_equality(trainer.callback_handler.callbacks  ,__UpperCAmelCase      )




  def 							UpperCAmelCase_	(    self      )    ->		int:
      lowerCAmelCase__ :  Optional[int]							      =					DEFAULT_CALLBACKS.copy() + [ProgressCallback]
      lowerCAmelCase__ :  List[Any]							      =					self.get_trainer()

      # We can add, pop, or remove by class name
      trainer.remove_callback(__UpperCAmelCase      )
      expected_callbacks.remove(__UpperCAmelCase      )
      self.check_callbacks_equality(trainer.callback_handler.callbacks  ,__UpperCAmelCase      )

      lowerCAmelCase__ :  Optional[int]							      =					self.get_trainer()
      lowerCAmelCase__ :  Optional[int]							      =					trainer.pop_callback(__UpperCAmelCase      )
      self.assertEqual(cb.__class__  ,__UpperCAmelCase      )
      self.check_callbacks_equality(trainer.callback_handler.callbacks  ,__UpperCAmelCase      )

      trainer.add_callback(__UpperCAmelCase      )
      expected_callbacks.insert(0  ,__UpperCAmelCase      )
      self.check_callbacks_equality(trainer.callback_handler.callbacks  ,__UpperCAmelCase      )

      # We can also add, pop, or remove by instance
      lowerCAmelCase__ :  Optional[int]							      =					self.get_trainer()
      lowerCAmelCase__ :  Any							      =					trainer.callback_handler.callbacks[0]
      trainer.remove_callback(__UpperCAmelCase      )
      expected_callbacks.remove(__UpperCAmelCase      )
      self.check_callbacks_equality(trainer.callback_handler.callbacks  ,__UpperCAmelCase      )

      lowerCAmelCase__ :  List[Any]							      =					self.get_trainer()
      lowerCAmelCase__ :  Dict							      =					trainer.callback_handler.callbacks[0]
      lowerCAmelCase__ :  List[Any]							      =					trainer.pop_callback(__UpperCAmelCase      )
      self.assertEqual(__UpperCAmelCase  ,__UpperCAmelCase      )
      self.check_callbacks_equality(trainer.callback_handler.callbacks  ,__UpperCAmelCase      )

      trainer.add_callback(__UpperCAmelCase      )
      expected_callbacks.insert(0  ,__UpperCAmelCase      )
      self.check_callbacks_equality(trainer.callback_handler.callbacks  ,__UpperCAmelCase      )






  def 							UpperCAmelCase_	(    self      )    ->		Optional[Any]:
      import warnings

      # XXX: for now ignore scatter_gather warnings in this test since it's not relevant to what's being tested
      warnings.simplefilter(action="""ignore"""  ,category=__UpperCAmelCase      )

      lowerCAmelCase__ :  List[Any]							      =					self.get_trainer(callbacks=[MyTestTrainerCallback]      )
      trainer.train()
      lowerCAmelCase__ :  str							      =					trainer.callback_handler.callbacks[-2].events
      self.assertEqual(__UpperCAmelCase  ,self.get_expected_events(__UpperCAmelCase      )      )

      # Independent log/save/eval
      lowerCAmelCase__ :  Union[str, Any]							      =					self.get_trainer(callbacks=[MyTestTrainerCallback]  ,logging_steps=5      )
      trainer.train()
      lowerCAmelCase__ :  List[str]							      =					trainer.callback_handler.callbacks[-2].events
      self.assertEqual(__UpperCAmelCase  ,self.get_expected_events(__UpperCAmelCase      )      )

      lowerCAmelCase__ :  List[Any]							      =					self.get_trainer(callbacks=[MyTestTrainerCallback]  ,save_steps=5      )
      trainer.train()
      lowerCAmelCase__ :  str							      =					trainer.callback_handler.callbacks[-2].events
      self.assertEqual(__UpperCAmelCase  ,self.get_expected_events(__UpperCAmelCase      )      )

      lowerCAmelCase__ :  Tuple							      =					self.get_trainer(callbacks=[MyTestTrainerCallback]  ,eval_steps=5  ,evaluation_strategy="""steps"""      )
      trainer.train()
      lowerCAmelCase__ :  Tuple							      =					trainer.callback_handler.callbacks[-2].events
      self.assertEqual(__UpperCAmelCase  ,self.get_expected_events(__UpperCAmelCase      )      )

      lowerCAmelCase__ :  int							      =					self.get_trainer(callbacks=[MyTestTrainerCallback]  ,evaluation_strategy="""epoch"""      )
      trainer.train()
      lowerCAmelCase__ :  Any							      =					trainer.callback_handler.callbacks[-2].events
      self.assertEqual(__UpperCAmelCase  ,self.get_expected_events(__UpperCAmelCase      )      )

      # A bit of everything
      lowerCAmelCase__ :  List[str]							      =					self.get_trainer(
          callbacks=[MyTestTrainerCallback]  ,logging_steps=3  ,save_steps=10  ,eval_steps=5  ,evaluation_strategy="""steps"""  ,)
      trainer.train()
      lowerCAmelCase__ :  Optional[Any]							      =					trainer.callback_handler.callbacks[-2].events
      self.assertEqual(__UpperCAmelCase  ,self.get_expected_events(__UpperCAmelCase      )      )

      # warning should be emitted for duplicated callbacks
      with patch("""transformers.trainer_callback.logger.warning"""      ) as warn_mock:
          lowerCAmelCase__ :  Union[str, Any]							      =					self.get_trainer(
              callbacks=[MyTestTrainerCallback, MyTestTrainerCallback]  ,)
          assert str(__UpperCAmelCase      ) in warn_mock.call_args[0][0]







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


__snake_case						:str      		=     logging.get_logger(__name__)

__snake_case						:int      		=     {'''vocab_file''': '''vocab.txt'''}

__snake_case						:List[Any]      		=     {
    '''vocab_file''': {
        '''YituTech/conv-bert-base''': '''https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt''',
        '''YituTech/conv-bert-medium-small''': (
            '''https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt'''
        ),
        '''YituTech/conv-bert-small''': '''https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt''',
    }
}

__snake_case						:List[str]      		=     {
    '''YituTech/conv-bert-base''': 512,
    '''YituTech/conv-bert-medium-small''': 512,
    '''YituTech/conv-bert-small''': 512,
}


__snake_case						:Optional[int]      		=     {
    '''YituTech/conv-bert-base''': {'''do_lower_case''': True},
    '''YituTech/conv-bert-medium-small''': {'''do_lower_case''': True},
    '''YituTech/conv-bert-small''': {'''do_lower_case''': True},
}

class     _A       (	__UpperCAmelCase    ):
       UpperCamelCase__  :    Optional[int]											=						VOCAB_FILES_NAMES
       UpperCamelCase__  :    str											=						PRETRAINED_VOCAB_FILES_MAP
       UpperCamelCase__  :    Optional[Any]											=						PRETRAINED_INIT_CONFIGURATION
       UpperCamelCase__  :    Optional[int]											=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       UpperCamelCase__  :    int											=						ConvBertTokenizer



       def __init__(		self							:     Optional[Any]     ,		__SCREAMING_SNAKE_CASE							:     Optional[Any]=None     ,		__SCREAMING_SNAKE_CASE							:     Dict=None     ,		__SCREAMING_SNAKE_CASE							:     Tuple=True     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]="[UNK]"     ,		__SCREAMING_SNAKE_CASE							:     int="[SEP]"     ,		__SCREAMING_SNAKE_CASE							:     List[Any]="[PAD]"     ,		__SCREAMING_SNAKE_CASE							:     int="[CLS]"     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]="[MASK]"     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]=True     ,		__SCREAMING_SNAKE_CASE							:     Union[str, Any]=None     ,		**__SCREAMING_SNAKE_CASE							:     Dict     ,		):


           '''simple docstring'''

           super().__init__(
               __SCREAMING_SNAKE_CASE     ,		tokenizer_file=__SCREAMING_SNAKE_CASE     ,		do_lower_case=__SCREAMING_SNAKE_CASE     ,		unk_token=__SCREAMING_SNAKE_CASE     ,		sep_token=__SCREAMING_SNAKE_CASE     ,		pad_token=__SCREAMING_SNAKE_CASE     ,		cls_token=__SCREAMING_SNAKE_CASE     ,		mask_token=__SCREAMING_SNAKE_CASE     ,		tokenize_chinese_chars=__SCREAMING_SNAKE_CASE     ,		strip_accents=__SCREAMING_SNAKE_CASE     ,		**__SCREAMING_SNAKE_CASE     ,		)

           __a						       =				json.loads(self.backend_tokenizer.normalizer.__getstate__())
           if (
               normalizer_state.get('''lowercase'''     ,		__SCREAMING_SNAKE_CASE) != do_lower_case
               or normalizer_state.get('''strip_accents'''     ,		__SCREAMING_SNAKE_CASE) != strip_accents
               or normalizer_state.get('''handle_chinese_chars'''     ,		__SCREAMING_SNAKE_CASE) != tokenize_chinese_chars
           ):
               __a						       =				getattr(__SCREAMING_SNAKE_CASE     ,		normalizer_state.pop('''type'''))
               __a						       =				do_lower_case
               __a						       =				strip_accents
               __a						       =				tokenize_chinese_chars
               __a						       =				normalizer_class(**__SCREAMING_SNAKE_CASE)

           __a						       =				do_lower_case



       def 		_lowerCamelCase			(		self							:     int     ,		__SCREAMING_SNAKE_CASE							:     Dict     ,		__SCREAMING_SNAKE_CASE							:     Any=None):


           '''simple docstring'''

           __a						       =				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

           return output



       def 		_lowerCamelCase			(		self							:     str     ,		__SCREAMING_SNAKE_CASE							:     List[int]     ,		__SCREAMING_SNAKE_CASE							:     Optional[List[int]] = None):


           '''simple docstring'''

           __a						       =				[self.sep_token_id]
           __a						       =				[self.cls_token_id]
           if token_ids_a is None:
               return len(cls + token_ids_a + sep) * [0]
           return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]





       def 		_lowerCamelCase			(		self							:     Optional[Any]     ,		__SCREAMING_SNAKE_CASE							:     str     ,		__SCREAMING_SNAKE_CASE							:     Optional[str] = None):


           '''simple docstring'''

           __a						       =				self._tokenizer.model.save(__SCREAMING_SNAKE_CASE     ,		name=__SCREAMING_SNAKE_CASE)
           return tuple(__SCREAMING_SNAKE_CASE)





