



"""simple docstring"""
import shutil
import tempfile
import unittest
from unittest.mock import patch

from transformers import (
    DefaultFlowCallback,
    IntervalStrategy,
    PrinterCallback,
    ProgressCallback,
    Trainer,
    TrainerCallback,
    TrainingArguments,
    is_torch_available,
)
from transformers.testing_utils import require_torch


if is_torch_available():
   from transformers.trainer import DEFAULT_CALLBACKS

   from .test_trainer import RegressionDataset, RegressionModelConfig, RegressionPreTrainedModel
class     a		(     SCREAMING_SNAKE_CASE_				):



  def __init__(  self       :    int						)   ->    Union[str, Any]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=[]



  def 		lowerCamelCase__				(  self       :    List[str]				,						lowerCAmelCase       :    Union[str, Any]				,						lowerCAmelCase       :    Optional[Any]				,						lowerCAmelCase       :    Optional[int]				,						**lowerCAmelCase       :    Optional[int]						)   ->    Optional[int]:


     '''simple docstring'''




     self.events.append("""on_init_end"""						)



  def 		lowerCamelCase__				(  self       :    Optional[Any]				,						lowerCAmelCase       :    Union[str, Any]				,						lowerCAmelCase       :    str				,						lowerCAmelCase       :    List[str]				,						**lowerCAmelCase       :    Dict						)   ->    int:


     '''simple docstring'''




     self.events.append("""on_train_begin"""						)



  def 		lowerCamelCase__				(  self       :    Union[str, Any]				,						lowerCAmelCase       :    Dict				,						lowerCAmelCase       :    List[Any]				,						lowerCAmelCase       :    List[str]				,						**lowerCAmelCase       :    Optional[Any]						)   ->    Optional[Any]:


     '''simple docstring'''




     self.events.append("""on_train_end"""						)



  def 		lowerCamelCase__				(  self       :    List[str]				,						lowerCAmelCase       :    Any				,						lowerCAmelCase       :    str				,						lowerCAmelCase       :    List[str]				,						**lowerCAmelCase       :    Optional[int]						)   ->    Dict:


     '''simple docstring'''




     self.events.append("""on_epoch_begin"""						)



  def 		lowerCamelCase__				(  self       :    Optional[Any]				,						lowerCAmelCase       :    List[str]				,						lowerCAmelCase       :    str				,						lowerCAmelCase       :    Any				,						**lowerCAmelCase       :    Dict						)   ->    Dict:


     '''simple docstring'''




     self.events.append("""on_epoch_end"""						)



  def 		lowerCamelCase__				(  self       :    Optional[int]				,						lowerCAmelCase       :    Tuple				,						lowerCAmelCase       :    Optional[Any]				,						lowerCAmelCase       :    int				,						**lowerCAmelCase       :    Optional[Any]						)   ->    Optional[int]:


     '''simple docstring'''




     self.events.append("""on_step_begin"""						)



  def 		lowerCamelCase__				(  self       :    Dict				,						lowerCAmelCase       :    Any				,						lowerCAmelCase       :    Dict				,						lowerCAmelCase       :    int				,						**lowerCAmelCase       :    Union[str, Any]						)   ->    Union[str, Any]:


     '''simple docstring'''




     self.events.append("""on_step_end"""						)



  def 		lowerCamelCase__				(  self       :    Optional[Any]				,						lowerCAmelCase       :    Any				,						lowerCAmelCase       :    Optional[Any]				,						lowerCAmelCase       :    Tuple				,						**lowerCAmelCase       :    Union[str, Any]						)   ->    Optional[int]:


     '''simple docstring'''




     self.events.append("""on_evaluate"""						)



  def 		lowerCamelCase__				(  self       :    int				,						lowerCAmelCase       :    Optional[Any]				,						lowerCAmelCase       :    Optional[Any]				,						lowerCAmelCase       :    str				,						**lowerCAmelCase       :    Any						)   ->    int:


     '''simple docstring'''




     self.events.append("""on_predict"""						)



  def 		lowerCamelCase__				(  self       :    Union[str, Any]				,						lowerCAmelCase       :    Optional[int]				,						lowerCAmelCase       :    Tuple				,						lowerCAmelCase       :    List[str]				,						**lowerCAmelCase       :    int						)   ->    Any:


     '''simple docstring'''




     self.events.append("""on_save"""						)



  def 		lowerCamelCase__				(  self       :    Tuple				,						lowerCAmelCase       :    Any				,						lowerCAmelCase       :    int				,						lowerCAmelCase       :    Optional[int]				,						**lowerCAmelCase       :    List[str]						)   ->    Any:


     '''simple docstring'''




     self.events.append("""on_log"""						)





  def 		lowerCamelCase__				(  self       :    Tuple				,						lowerCAmelCase       :    Union[str, Any]				,						lowerCAmelCase       :    Any				,						lowerCAmelCase       :    List[Any]				,						**lowerCAmelCase       :    Optional[Any]						)   ->    Optional[int]:


     '''simple docstring'''




     self.events.append("""on_prediction_step"""						)







@require_torch
class     a		(     unittest.TestCase				):



  def 		lowerCamelCase__				(  self       :    Optional[Any]						)   ->    Dict:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       int   	=tempfile.mkdtemp()



  def 		lowerCamelCase__				(  self       :    List[str]						)   ->    Tuple:


     '''simple docstring'''




     shutil.rmtree(self.output_dir						)



  def 		lowerCamelCase__				(  self       :    Dict				,						lowerCAmelCase       :    Optional[int]=0				,						lowerCAmelCase       :    Union[str, Any]=0				,						lowerCAmelCase       :    Optional[int]=64				,						lowerCAmelCase       :    List[Any]=64				,						lowerCAmelCase       :    Any=None				,						lowerCAmelCase       :    Optional[int]=False				,						**lowerCAmelCase       :    List[str]						)   ->    Optional[int]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       Tuple   	=RegressionDataset(length=__UpperCAmelCase						)
     SCREAMING_SNAKE_CASE_:       Tuple   	=RegressionDataset(length=__UpperCAmelCase						)
     SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=RegressionModelConfig(a=__UpperCAmelCase				,						b=__UpperCAmelCase						)
     SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=RegressionPreTrainedModel(__UpperCAmelCase						)

     SCREAMING_SNAKE_CASE_:       int   	=TrainingArguments(self.output_dir				,						disable_tqdm=__UpperCAmelCase				,						report_to=[]				,						**__UpperCAmelCase						)
     return Trainer(
         __UpperCAmelCase				,						__UpperCAmelCase				,						train_dataset=__UpperCAmelCase				,						eval_dataset=__UpperCAmelCase				,						callbacks=__UpperCAmelCase				,						)



  def 		lowerCamelCase__				(  self       :    Optional[int]				,						lowerCAmelCase       :    int				,						lowerCAmelCase       :    Union[str, Any]						)   ->    Dict:


     '''simple docstring'''




     self.assertEqual(len(__UpperCAmelCase						)				,						len(__UpperCAmelCase						)						)

     # Order doesn't matter
     SCREAMING_SNAKE_CASE_:       Optional[int]   	=sorted(__UpperCAmelCase				,						key=lambda lowerCAmelCase						: cb.__name__ if isinstance(__UpperCAmelCase				,						__UpperCAmelCase						) else cb.__class__.__name__						)
     SCREAMING_SNAKE_CASE_:       Optional[int]   	=sorted(__UpperCAmelCase				,						key=lambda lowerCAmelCase						: cb.__name__ if isinstance(__UpperCAmelCase				,						__UpperCAmelCase						) else cb.__class__.__name__						)

     for cba, cba in zip(__UpperCAmelCase				,						__UpperCAmelCase						):
        if isinstance(__UpperCAmelCase				,						__UpperCAmelCase						) and isinstance(__UpperCAmelCase				,						__UpperCAmelCase						):
           self.assertEqual(__UpperCAmelCase				,						__UpperCAmelCase						)
        elif isinstance(__UpperCAmelCase				,						__UpperCAmelCase						) and not isinstance(__UpperCAmelCase				,						__UpperCAmelCase						):
           self.assertEqual(__UpperCAmelCase				,						cba.__class__						)
        elif not isinstance(__UpperCAmelCase				,						__UpperCAmelCase						) and isinstance(__UpperCAmelCase				,						__UpperCAmelCase						):
           self.assertEqual(cba.__class__				,						__UpperCAmelCase						)
        else:
           self.assertEqual(__UpperCAmelCase				,						__UpperCAmelCase						)



  def 		lowerCamelCase__				(  self       :    Optional[int]				,						lowerCAmelCase       :    Tuple						)   ->    Union[str, Any]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       Any   	=["""on_init_end""", """on_train_begin"""]
     SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=0
     SCREAMING_SNAKE_CASE_:       Tuple   	=len(trainer.get_eval_dataloader()						)
     SCREAMING_SNAKE_CASE_:       Optional[int]   	=["""on_prediction_step"""] * len(trainer.get_eval_dataloader()						) + ["""on_log""", """on_evaluate"""]
     for _ in range(trainer.state.num_train_epochs						):
        expected_events.append("""on_epoch_begin"""						)
        for _ in range(__UpperCAmelCase						):
           step += 1
           expected_events += ["on_step_begin", "on_step_end"]
           if step % trainer.args.logging_steps == 0:
              expected_events.append("""on_log"""						)
           if trainer.args.evaluation_strategy == IntervalStrategy.STEPS and step % trainer.args.eval_steps == 0:
              expected_events += evaluation_events.copy()
           if step % trainer.args.save_steps == 0:
              expected_events.append("""on_save"""						)
        expected_events.append("""on_epoch_end"""						)
        if trainer.args.evaluation_strategy == IntervalStrategy.EPOCH:
           expected_events += evaluation_events.copy()
     expected_events += ["on_log", "on_train_end"]
     return expected_events



  def 		lowerCamelCase__				(  self       :    Any						)   ->    Dict:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       Any   	=self.get_trainer()
     SCREAMING_SNAKE_CASE_:       Tuple   	=DEFAULT_CALLBACKS.copy() + [ProgressCallback]
     self.check_callbacks_equality(trainer.callback_handler.callbacks				,						__UpperCAmelCase						)

     # Callbacks passed at init are added to the default callbacks
     SCREAMING_SNAKE_CASE_:       Optional[Any]   	=self.get_trainer(callbacks=[MyTestTrainerCallback]						)
     expected_callbacks.append(__UpperCAmelCase						)
     self.check_callbacks_equality(trainer.callback_handler.callbacks				,						__UpperCAmelCase						)

     # TrainingArguments.disable_tqdm controls if use ProgressCallback or PrinterCallback
     SCREAMING_SNAKE_CASE_:       Tuple   	=self.get_trainer(disable_tqdm=__UpperCAmelCase						)
     SCREAMING_SNAKE_CASE_:       int   	=DEFAULT_CALLBACKS.copy() + [PrinterCallback]
     self.check_callbacks_equality(trainer.callback_handler.callbacks				,						__UpperCAmelCase						)



  def 		lowerCamelCase__				(  self       :    Optional[Any]						)   ->    int:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       Optional[int]   	=DEFAULT_CALLBACKS.copy() + [ProgressCallback]
     SCREAMING_SNAKE_CASE_:       List[Any]   	=self.get_trainer()

     # We can add, pop, or remove by class name
     trainer.remove_callback(__UpperCAmelCase						)
     expected_callbacks.remove(__UpperCAmelCase						)
     self.check_callbacks_equality(trainer.callback_handler.callbacks				,						__UpperCAmelCase						)

     SCREAMING_SNAKE_CASE_:       Optional[int]   	=self.get_trainer()
     SCREAMING_SNAKE_CASE_:       Optional[int]   	=trainer.pop_callback(__UpperCAmelCase						)
     self.assertEqual(cb.__class__				,						__UpperCAmelCase						)
     self.check_callbacks_equality(trainer.callback_handler.callbacks				,						__UpperCAmelCase						)

     trainer.add_callback(__UpperCAmelCase						)
     expected_callbacks.insert(0				,						__UpperCAmelCase						)
     self.check_callbacks_equality(trainer.callback_handler.callbacks				,						__UpperCAmelCase						)

     # We can also add, pop, or remove by instance
     SCREAMING_SNAKE_CASE_:       Optional[int]   	=self.get_trainer()
     SCREAMING_SNAKE_CASE_:       Any   	=trainer.callback_handler.callbacks[0]
     trainer.remove_callback(__UpperCAmelCase						)
     expected_callbacks.remove(__UpperCAmelCase						)
     self.check_callbacks_equality(trainer.callback_handler.callbacks				,						__UpperCAmelCase						)

     SCREAMING_SNAKE_CASE_:       List[Any]   	=self.get_trainer()
     SCREAMING_SNAKE_CASE_:       Dict   	=trainer.callback_handler.callbacks[0]
     SCREAMING_SNAKE_CASE_:       List[Any]   	=trainer.pop_callback(__UpperCAmelCase						)
     self.assertEqual(__UpperCAmelCase				,						__UpperCAmelCase						)
     self.check_callbacks_equality(trainer.callback_handler.callbacks				,						__UpperCAmelCase						)

     trainer.add_callback(__UpperCAmelCase						)
     expected_callbacks.insert(0				,						__UpperCAmelCase						)
     self.check_callbacks_equality(trainer.callback_handler.callbacks				,						__UpperCAmelCase						)





  def 		lowerCamelCase__				(  self       :    Optional[int]						)   ->    Optional[Any]:


     '''simple docstring'''




     import warnings

     # XXX: for now ignore scatter_gather warnings in this test since it's not relevant to what's being tested
     warnings.simplefilter(action="""ignore"""				,						category=__UpperCAmelCase						)

     SCREAMING_SNAKE_CASE_:       List[Any]   	=self.get_trainer(callbacks=[MyTestTrainerCallback]						)
     trainer.train()
     SCREAMING_SNAKE_CASE_:       str   	=trainer.callback_handler.callbacks[-2].events
     self.assertEqual(__UpperCAmelCase				,						self.get_expected_events(__UpperCAmelCase						)						)

     # Independent log/save/eval
     SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=self.get_trainer(callbacks=[MyTestTrainerCallback]				,						logging_steps=5						)
     trainer.train()
     SCREAMING_SNAKE_CASE_:       List[str]   	=trainer.callback_handler.callbacks[-2].events
     self.assertEqual(__UpperCAmelCase				,						self.get_expected_events(__UpperCAmelCase						)						)

     SCREAMING_SNAKE_CASE_:       List[Any]   	=self.get_trainer(callbacks=[MyTestTrainerCallback]				,						save_steps=5						)
     trainer.train()
     SCREAMING_SNAKE_CASE_:       str   	=trainer.callback_handler.callbacks[-2].events
     self.assertEqual(__UpperCAmelCase				,						self.get_expected_events(__UpperCAmelCase						)						)

     SCREAMING_SNAKE_CASE_:       Tuple   	=self.get_trainer(callbacks=[MyTestTrainerCallback]				,						eval_steps=5				,						evaluation_strategy="""steps"""						)
     trainer.train()
     SCREAMING_SNAKE_CASE_:       Tuple   	=trainer.callback_handler.callbacks[-2].events
     self.assertEqual(__UpperCAmelCase				,						self.get_expected_events(__UpperCAmelCase						)						)

     SCREAMING_SNAKE_CASE_:       int   	=self.get_trainer(callbacks=[MyTestTrainerCallback]				,						evaluation_strategy="""epoch"""						)
     trainer.train()
     SCREAMING_SNAKE_CASE_:       Any   	=trainer.callback_handler.callbacks[-2].events
     self.assertEqual(__UpperCAmelCase				,						self.get_expected_events(__UpperCAmelCase						)						)

     # A bit of everything
     SCREAMING_SNAKE_CASE_:       List[str]   	=self.get_trainer(
         callbacks=[MyTestTrainerCallback]				,						logging_steps=3				,						save_steps=10				,						eval_steps=5				,						evaluation_strategy="""steps"""				,						)
     trainer.train()
     SCREAMING_SNAKE_CASE_:       Optional[Any]   	=trainer.callback_handler.callbacks[-2].events
     self.assertEqual(__UpperCAmelCase				,						self.get_expected_events(__UpperCAmelCase						)						)

     # warning should be emitted for duplicated callbacks
     with patch("""transformers.trainer_callback.logger.warning"""						) as warn_mock:
        SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=self.get_trainer(
            callbacks=[MyTestTrainerCallback, MyTestTrainerCallback]				,						)
        assert str(__UpperCAmelCase						) in warn_mock.call_args[0][0]








'''simple docstring'''





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase      						=	logging.get_logger(__name__)
_lowerCAmelCase      						=	{'''vocab_file''': '''spiece.model'''}

_lowerCAmelCase      						=	{
    '''vocab_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/spiece.model''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/spiece.model''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/spiece.model''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model''',
    }
}

_lowerCAmelCase      						=	{
    '''albert-base-v1''': 512,
    '''albert-large-v1''': 512,
    '''albert-xlarge-v1''': 512,
    '''albert-xxlarge-v1''': 512,
    '''albert-base-v2''': 512,
    '''albert-large-v2''': 512,
    '''albert-xlarge-v2''': 512,
    '''albert-xxlarge-v2''': 512,
}

_lowerCAmelCase      						=	'''▁'''






class        lowerCAmelCase_(				SCREAMING_SNAKE_CASE_     ):
  '''simple docstring'''




  __lowercase							:      Any				=      VOCAB_FILES_NAMES
  __lowercase							:      Optional[Any]				=      PRETRAINED_VOCAB_FILES_MAP
  __lowercase							:      int				=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




  def __init__(    self  ,__UpperCAmelCase  ,__UpperCAmelCase=True  ,__UpperCAmelCase=True  ,__UpperCAmelCase=False  ,__UpperCAmelCase="[CLS]"  ,__UpperCAmelCase="[SEP]"  ,__UpperCAmelCase="<unk>"  ,__UpperCAmelCase="[SEP]"  ,__UpperCAmelCase="<pad>"  ,__UpperCAmelCase="[CLS]"  ,__UpperCAmelCase="[MASK]"  ,__UpperCAmelCase = None  ,**__UpperCAmelCase  ,)    ->		None:
      # Mask token behave like a normal word, i.e. include the space before it and
      # is included in the raw text, there should be a match in a non-normalized sentence.
      lowerCAmelCase__ :  Tuple							      =					(
          AddedToken(__UpperCAmelCase  ,lstrip=__UpperCAmelCase  ,rstrip=__UpperCAmelCase  ,normalized=__UpperCAmelCase      )
          if isinstance(__UpperCAmelCase  ,__UpperCAmelCase      )
          else mask_token
      )

      lowerCAmelCase__ :  Any							      =					{} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          do_lower_case=__UpperCAmelCase  ,remove_space=__UpperCAmelCase  ,keep_accents=__UpperCAmelCase  ,bos_token=__UpperCAmelCase  ,eos_token=__UpperCAmelCase  ,unk_token=__UpperCAmelCase  ,sep_token=__UpperCAmelCase  ,pad_token=__UpperCAmelCase  ,cls_token=__UpperCAmelCase  ,mask_token=__UpperCAmelCase  ,sp_model_kwargs=self.sp_model_kwargs  ,**__UpperCAmelCase  ,)

      lowerCAmelCase__ :  str							      =					do_lower_case
      lowerCAmelCase__ :  int							      =					remove_space
      lowerCAmelCase__ :  Tuple							      =					keep_accents
      lowerCAmelCase__ :  Any							      =					vocab_file

      lowerCAmelCase__ :  Tuple							      =					spm.SentencePieceProcessor(**self.sp_model_kwargs      )
      self.sp_model.Load(__UpperCAmelCase      )




  @property
  def 							UpperCAmelCase_	(    self      )    ->		Optional[int]:
      return len(self.sp_model      )




  def 							UpperCAmelCase_	(    self      )    ->		Optional[Any]:
      lowerCAmelCase__ :  Union[str, Any]							      =					{self.convert_ids_to_tokens(__UpperCAmelCase      ): i for i in range(self.vocab_size      )}
      vocab.update(self.added_tokens_encoder      )
      return vocab




  def __getstate__(    self      )    ->		Any:
      lowerCAmelCase__ :  Optional[Any]							      =					self.__dict__.copy()
      lowerCAmelCase__ :  Optional[Any]							      =					None
      return state




  def __setstate__(    self  ,__UpperCAmelCase      )    ->		List[Any]:
      lowerCAmelCase__ :  List[str]							      =					d

      # for backward compatibility
      if not hasattr(self  ,"""sp_model_kwargs"""      ):
          lowerCAmelCase__ :  Union[str, Any]							      =					{}

      lowerCAmelCase__ :  Any							      =					spm.SentencePieceProcessor(**self.sp_model_kwargs      )
      self.sp_model.Load(self.vocab_file      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase      )    ->		Optional[int]:
      if self.remove_space:
          lowerCAmelCase__ :  int							      =					""" """.join(inputs.strip().split()      )
      else:
          lowerCAmelCase__ :  str							      =					inputs
      lowerCAmelCase__ :  Tuple							      =					outputs.replace("""``"""  ,"""\""""      ).replace("""''"""  ,"""\""""      )

      if not self.keep_accents:
          lowerCAmelCase__ :  Any							      =					unicodedata.normalize("""NFKD"""  ,__UpperCAmelCase      )
          lowerCAmelCase__ :  Dict							      =					"""""".join([c for c in outputs if not unicodedata.combining(__UpperCAmelCase      )]      )
      if self.do_lower_case:
          lowerCAmelCase__ :  Tuple							      =					outputs.lower()

      return outputs




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase      )    ->		List[str]:
      lowerCAmelCase__ :  List[str]							      =					self.preprocess_text(__UpperCAmelCase      )
      lowerCAmelCase__ :  Optional[int]							      =					self.sp_model.encode(__UpperCAmelCase  ,out_type=__UpperCAmelCase      )
      lowerCAmelCase__ :  str							      =					[]
      for piece in pieces:
          if len(__UpperCAmelCase      ) > 1 and piece[-1] == str(""","""      ) and piece[-2].isdigit():
              lowerCAmelCase__ :  List[Any]							      =					self.sp_model.EncodeAsPieces(piece[:-1].replace(__UpperCAmelCase  ,""""""      )      )
              if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                  if len(cur_pieces[0]      ) == 1:
                      lowerCAmelCase__ :  str							      =					cur_pieces[1:]
                  else:
                      lowerCAmelCase__ :  int							      =					cur_pieces[0][1:]
              cur_pieces.append(piece[-1]      )
              new_pieces.extend(__UpperCAmelCase      )
          else:
              new_pieces.append(__UpperCAmelCase      )

      return new_pieces




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase      )    ->		List[str]:
      return self.sp_model.PieceToId(__UpperCAmelCase      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase      )    ->		Any:
      return self.sp_model.IdToPiece(__UpperCAmelCase      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase      )    ->		str:
      lowerCAmelCase__ :  str							      =					[]
      lowerCAmelCase__ :  Tuple							      =					""""""
      lowerCAmelCase__ :  Tuple							      =					False
      for token in tokens:
          # make sure that special tokens are not decoded using sentencepiece model
          if token in self.all_special_tokens:
              if not prev_is_special:
                  out_string += " "
              out_string += self.sp_model.decode(__UpperCAmelCase      ) + token
              lowerCAmelCase__ :  Union[str, Any]							      =					True
              lowerCAmelCase__ :  List[str]							      =					[]
          else:
              current_sub_tokens.append(__UpperCAmelCase      )
              lowerCAmelCase__ :  Optional[Any]							      =					False
      out_string += self.sp_model.decode(__UpperCAmelCase      )
      return out_string.strip()




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase = None      )    ->		List[int]:
      lowerCAmelCase__ :  int							      =					[self.sep_token_id]
      lowerCAmelCase__ :  Dict							      =					[self.cls_token_id]
      if token_ids_a is None:
          return cls + token_ids_a + sep
      return cls + token_ids_a + sep + token_ids_a + sep




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase = None  ,__UpperCAmelCase = False      )    ->		List[int]:

      if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=__UpperCAmelCase  ,token_ids_a=__UpperCAmelCase  ,already_has_special_tokens=__UpperCAmelCase      )

      if token_ids_a is not None:
          return [1] + ([0] * len(__UpperCAmelCase      )) + [1] + ([0] * len(__UpperCAmelCase      )) + [1]
      return [1] + ([0] * len(__UpperCAmelCase      )) + [1]




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase = None      )    ->		List[int]:
      lowerCAmelCase__ :  List[str]							      =					[self.sep_token_id]
      lowerCAmelCase__ :  Tuple							      =					[self.cls_token_id]

      if token_ids_a is None:
          return len(cls + token_ids_a + sep      ) * [0]
      return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]






  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase = None      )    ->		Tuple[str]:
      if not os.path.isdir(__UpperCAmelCase      ):
          logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""      )
          return
      lowerCAmelCase__ :  int							      =					os.path.join(
          __UpperCAmelCase  ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]      )

      if os.path.abspath(self.vocab_file      ) != os.path.abspath(__UpperCAmelCase      ) and os.path.isfile(self.vocab_file      ):
          copyfile(self.vocab_file  ,__UpperCAmelCase      )
      elif not os.path.isfile(self.vocab_file      ):
          with open(__UpperCAmelCase  ,"""wb"""      ) as fi:
              lowerCAmelCase__ :  List[Any]							      =					self.sp_model.serialized_model_proto()
              fi.write(__UpperCAmelCase      )

      return (out_vocab_file,)


