import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


UpperCamelCase__											=					logging.get_logger(__name__)

UpperCamelCase__											=					{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCamelCase__											=					{
    'vocab_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/vocab.txt',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/vocab.txt',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'
        ),
        'bert-base-multilingual-cased': 'https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt',
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'
        ),
        'bert-base-german-dbmdz-cased': 'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt',
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'
        ),
        'bert-base-multilingual-cased': (
            'https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-cased': (
            'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'
        ),
    },
}

UpperCamelCase__											=					{
    'bert-base-uncased': 5_1_2,
    'bert-large-uncased': 5_1_2,
    'bert-base-cased': 5_1_2,
    'bert-large-cased': 5_1_2,
    'bert-base-multilingual-uncased': 5_1_2,
    'bert-base-multilingual-cased': 5_1_2,
    'bert-base-chinese': 5_1_2,
    'bert-base-german-cased': 5_1_2,
    'bert-large-uncased-whole-word-masking': 5_1_2,
    'bert-large-cased-whole-word-masking': 5_1_2,
    'bert-large-uncased-whole-word-masking-finetuned-squad': 5_1_2,
    'bert-large-cased-whole-word-masking-finetuned-squad': 5_1_2,
    'bert-base-cased-finetuned-mrpc': 5_1_2,
    'bert-base-german-dbmdz-cased': 5_1_2,
    'bert-base-german-dbmdz-uncased': 5_1_2,
    'TurkuNLP/bert-base-finnish-cased-v1': 5_1_2,
    'TurkuNLP/bert-base-finnish-uncased-v1': 5_1_2,
    'wietsedv/bert-base-dutch-cased': 5_1_2,
}

UpperCamelCase__											=					{
    'bert-base-uncased': {'do_lower_case': True},
    'bert-large-uncased': {'do_lower_case': True},
    'bert-base-cased': {'do_lower_case': False},
    'bert-large-cased': {'do_lower_case': False},
    'bert-base-multilingual-uncased': {'do_lower_case': True},
    'bert-base-multilingual-cased': {'do_lower_case': False},
    'bert-base-chinese': {'do_lower_case': False},
    'bert-base-german-cased': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking-finetuned-squad': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking-finetuned-squad': {'do_lower_case': False},
    'bert-base-cased-finetuned-mrpc': {'do_lower_case': False},
    'bert-base-german-dbmdz-cased': {'do_lower_case': False},
    'bert-base-german-dbmdz-uncased': {'do_lower_case': True},
    'TurkuNLP/bert-base-finnish-cased-v1': {'do_lower_case': False},
    'TurkuNLP/bert-base-finnish-uncased-v1': {'do_lower_case': True},
    'wietsedv/bert-base-dutch-cased': {'do_lower_case': False},
}





class      A							(				UpperCAmelCase_     ):
   __UpperCAmelCase : str      =			VOCAB_FILES_NAMES
   __UpperCAmelCase : Optional[Any]      =			PRETRAINED_VOCAB_FILES_MAP
   __UpperCAmelCase : List[Any]      =			PRETRAINED_INIT_CONFIGURATION
   __UpperCAmelCase : Optional[int]      =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCAmelCase : str      =			BertTokenizer


   def __init__(self     :						Any       ,  __UpperCAmelCase     :						Optional[int]=None       ,  __UpperCAmelCase     :						Any=None       ,  __UpperCAmelCase     :						Dict=True       ,  __UpperCAmelCase     :						str="[UNK]"       ,  __UpperCAmelCase     :						List[str]="[SEP]"       ,  __UpperCAmelCase     :						int="[PAD]"       ,  __UpperCAmelCase     :						List[Any]="[CLS]"       ,  __UpperCAmelCase     :						str="[MASK]"       ,  __UpperCAmelCase     :						List[str]=True       ,  __UpperCAmelCase     :						Tuple=None       ,  **__UpperCAmelCase     :						int       ,  )						->				List[Any]:
         """simple docstring"""



         super().__init__(
             __UpperCAmelCase       ,  tokenizer_file=__UpperCAmelCase       ,  do_lower_case=__UpperCAmelCase       ,  unk_token=__UpperCAmelCase       ,  sep_token=__UpperCAmelCase       ,  pad_token=__UpperCAmelCase       ,  cls_token=__UpperCAmelCase       ,  mask_token=__UpperCAmelCase       ,  tokenize_chinese_chars=__UpperCAmelCase       ,  strip_accents=__UpperCAmelCase       ,  **__UpperCAmelCase       ,  )

         UpperCAmelCase__								=  json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
         if (
             normalizer_state.get("lowercase"       ,  __UpperCAmelCase		) != do_lower_case
             or normalizer_state.get("strip_accents"       ,  __UpperCAmelCase		) != strip_accents
             or normalizer_state.get("handle_chinese_chars"       ,  __UpperCAmelCase		) != tokenize_chinese_chars
         ):
               UpperCAmelCase__								=  getattr(__UpperCAmelCase       ,  normalizer_state.pop("type"		)		)
               UpperCAmelCase__								=  do_lower_case
               UpperCAmelCase__								=  strip_accents
               UpperCAmelCase__								=  tokenize_chinese_chars
               UpperCAmelCase__								=  normalizer_class(**__UpperCAmelCase		)

         UpperCAmelCase__								=  do_lower_case


   def 			lowercase_      (self     :						int       ,  __UpperCAmelCase     :						str       ,  __UpperCAmelCase     :						List[str]=None		)						->				Dict:
         """simple docstring"""



         UpperCAmelCase__								=  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

         return output


   def 			lowercase_      (self     :						Dict       ,  __UpperCAmelCase     :						List[int]       ,  __UpperCAmelCase     :						Optional[List[int]] = None		)						->				List[int]:
         """simple docstring"""



         UpperCAmelCase__								=  [self.sep_token_id]
         UpperCAmelCase__								=  [self.cls_token_id]
         if token_ids_a is None:
               return len(cls + token_ids_a + sep		) * [0]
         return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]





   def 			lowercase_      (self     :						Union[str, Any]       ,  __UpperCAmelCase     :						str       ,  __UpperCAmelCase     :						Optional[str] = None		)						->				Tuple[str]:
         """simple docstring"""



         UpperCAmelCase__								=  self._tokenizer.model.save(__UpperCAmelCase       ,  name=__UpperCAmelCase		)
         return tuple(__UpperCAmelCase		)










'''simple docstring'''




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_albert import AlbertTokenizer
else:
  __SCREAMING_SNAKE_CASE     :List[Any]       =				None

__SCREAMING_SNAKE_CASE     :Union[str, Any]       =				logging.get_logger(__name__)
__SCREAMING_SNAKE_CASE     :List[str]       =				{'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

__SCREAMING_SNAKE_CASE     :List[Any]       =				{
    '''vocab_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/spiece.model''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/spiece.model''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/spiece.model''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json''',
    },
}

__SCREAMING_SNAKE_CASE     :Optional[Any]       =				{
    '''albert-base-v1''': 512,
    '''albert-large-v1''': 512,
    '''albert-xlarge-v1''': 512,
    '''albert-xxlarge-v1''': 512,
    '''albert-base-v2''': 512,
    '''albert-large-v2''': 512,
    '''albert-xlarge-v2''': 512,
    '''albert-xxlarge-v2''': 512,
}

__SCREAMING_SNAKE_CASE     :Optional[int]       =				'''▁'''







class   A_	(  lowerCAmelCase_			):
 _lowerCamelCase  : Optional[int]           =							VOCAB_FILES_NAMES
 _lowerCamelCase  : Any           =							PRETRAINED_VOCAB_FILES_MAP
 _lowerCamelCase  : Tuple           =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowerCamelCase  : int           =							AlbertTokenizer




 def __init__(					self       :    Optional[Any]	,			snake_case_       :    Optional[Any]=None	,			snake_case_       :    Optional[Any]=None	,			snake_case_       :    Optional[Any]=True	,			snake_case_       :    str=True	,			snake_case_       :    Tuple=False	,			snake_case_       :    List[Any]="[CLS]"	,			snake_case_       :    Union[str, Any]="[SEP]"	,			snake_case_       :    str="<unk>"	,			snake_case_       :    Union[str, Any]="[SEP]"	,			snake_case_       :    List[Any]="<pad>"	,			snake_case_       :    List[str]="[CLS]"	,			snake_case_       :    int="[MASK]"	,			**snake_case_       :    Any	,			):
      # Mask token behave like a normal word, i.e. include the space before it and
      # is included in the raw text, there should be a match in a non-normalized sentence.
      _UpperCAmelCase     							=   (
          AddedToken(snake_case_	,			lstrip=snake_case_	,			rstrip=snake_case_	,			normalized=snake_case_    )
          if isinstance(snake_case_	,			snake_case_    )
          else mask_token
      )

      super().__init__(
          snake_case_	,			tokenizer_file=snake_case_	,			do_lower_case=snake_case_	,			remove_space=snake_case_	,			keep_accents=snake_case_	,			bos_token=snake_case_	,			eos_token=snake_case_	,			unk_token=snake_case_	,			sep_token=snake_case_	,			pad_token=snake_case_	,			cls_token=snake_case_	,			mask_token=snake_case_	,			**snake_case_	,			)

      _UpperCAmelCase     							=   do_lower_case
      _UpperCAmelCase     							=   remove_space
      _UpperCAmelCase     							=   keep_accents
      _UpperCAmelCase     							=   vocab_file
      _UpperCAmelCase     							=   False if not self.vocab_file else True




 def      lowercase	(					self       :    Union[str, Any]	,			snake_case_       :    List[int]	,			snake_case_       :    Optional[List[int]] = None    ):
      _UpperCAmelCase     							=   [self.sep_token_id]
      _UpperCAmelCase     							=   [self.cls_token_id]
      if token_ids_a is None:
           return cls + token_ids_a + sep
      return cls + token_ids_a + sep + token_ids_a + sep




 def      lowercase	(					self       :    Dict	,			snake_case_       :    List[int]	,			snake_case_       :    Optional[List[int]] = None    ):
      _UpperCAmelCase     							=   [self.sep_token_id]
      _UpperCAmelCase     							=   [self.cls_token_id]

      if token_ids_a is None:
           return len(cls + token_ids_a + sep    ) * [0]
      return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]




 def      lowercase	(					self       :    Optional[Any]	,			snake_case_       :    str	,			snake_case_       :    Optional[str] = None    ):
      if not self.can_save_slow_tokenizer:
           raise ValueError(
               "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
               "tokenizer."    )

      if not os.path.isdir(snake_case_    ):
           logger.error(f'Vocabulary path ({save_directory}) should be a directory'    )
           return
      _UpperCAmelCase     							=   os.path.join(
          snake_case_	,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )

      if os.path.abspath(self.vocab_file    ) != os.path.abspath(snake_case_    ):
           copyfile(self.vocab_file	,			snake_case_    )

      return (out_vocab_file,)




