'''simple docstring'''


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


snake_case__	:     str      =    logging.get_logger(__name__)

snake_case__	:     Any      =    {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

snake_case__	:     Dict      =    {
    '''vocab_file''': {
        '''microsoft/layoutlm-base-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt'''
        ),
        '''microsoft/layoutlm-large-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''microsoft/layoutlm-base-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json'''
        ),
        '''microsoft/layoutlm-large-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json'''
        ),
    },
}

snake_case__	:     Dict      =    {
    '''microsoft/layoutlm-base-uncased''': 512,
    '''microsoft/layoutlm-large-uncased''': 512,
}

snake_case__	:     Optional[Any]      =    {
    '''microsoft/layoutlm-base-uncased''': {'''do_lower_case''': True},
    '''microsoft/layoutlm-large-uncased''': {'''do_lower_case''': True},
}






class 	__SCREAMING_SNAKE_CASE		(			lowerCamelCase_			):







  '''simple docstring'''






  lowerCamelCase_						:Optional[Any]		       =						VOCAB_FILES_NAMES
  lowerCamelCase_						:Dict		       =						PRETRAINED_VOCAB_FILES_MAP
  lowerCamelCase_						:Tuple		       =						PRETRAINED_INIT_CONFIGURATION
  lowerCamelCase_						:int		       =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowerCamelCase_						:Optional[int]		       =						LayoutLMTokenizer


  def __init__(					self ,						snake_case_=None ,						snake_case_=None ,						snake_case_=True ,						snake_case_="[UNK]" ,						snake_case_="[SEP]" ,						snake_case_="[PAD]" ,						snake_case_="[CLS]" ,						snake_case_="[MASK]" ,						snake_case_=True ,						snake_case_=None ,						**snake_case_ ,						):




       '''simple docstring'''

       super().__init__(
           snake_case_ ,						tokenizer_file=snake_case_ ,						do_lower_case=snake_case_ ,						unk_token=snake_case_ ,						sep_token=snake_case_ ,						pad_token=snake_case_ ,						cls_token=snake_case_ ,						mask_token=snake_case_ ,						tokenize_chinese_chars=snake_case_ ,						strip_accents=snake_case_ ,						**snake_case_ ,						)

       UpperCAmelCase_				:  Any  = json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
       if (
           normalizer_state.get('lowercase' ,						snake_case_		) != do_lower_case
           or normalizer_state.get('strip_accents' ,						snake_case_		) != strip_accents
           or normalizer_state.get('handle_chinese_chars' ,						snake_case_		) != tokenize_chinese_chars
       ):
            UpperCAmelCase_				:  Tuple  = getattr(snake_case_ ,						normalizer_state.pop('type'		)		)
            UpperCAmelCase_				:  Union[str, Any]  = do_lower_case
            UpperCAmelCase_				:  Tuple  = strip_accents
            UpperCAmelCase_				:  Dict  = tokenize_chinese_chars
            UpperCAmelCase_				:  List[str]  = normalizer_class(**snake_case_		)

       UpperCAmelCase_				:  Optional[Any]  = do_lower_case


  def _UpperCamelCase	(					self ,						snake_case_ ,						snake_case_=None		):




       '''simple docstring'''

       UpperCAmelCase_				:  List[str]  = [self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
            output += token_ids_a + [self.sep_token_id]

       return output


  def _UpperCamelCase	(					self ,						snake_case_ ,						snake_case_ = None		):




       '''simple docstring'''

       UpperCAmelCase_				:  Optional[Any]  = [self.sep_token_id]
       UpperCAmelCase_				:  Union[str, Any]  = [self.cls_token_id]
       if token_ids_a is None:
            return len(cls + token_ids_a + sep		) * [0]
       return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]






  def _UpperCamelCase	(					self ,						snake_case_ ,						snake_case_ = None		):




       '''simple docstring'''

       UpperCAmelCase_				:  Tuple  = self._tokenizer.model.save(snake_case_ ,						name=snake_case_		)
       return tuple(snake_case_		)







'''simple docstring'''




_A										= '\n# Transformers installation\n! pip install transformers datasets\n# To install from source instead of the last release, comment the command above and uncomment the following one.\n# ! pip install git+https://github.com/huggingface/transformers.git\n'

_A										= [{'type': 'code', 'content': INSTALL_CONTENT}]
_A										= {
    '{processor_class}': 'FakeProcessorClass',
    '{model_class}': 'FakeModelClass',
    '{object_class}': 'FakeObjectClass',
}




