

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


_lowerCamelCase							:  int     =				logging.get_logger(__name__)

_lowerCamelCase							:  Optional[Any]     =				{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_lowerCamelCase							:  List[str]     =				{
    '''vocab_file''': {
        '''microsoft/layoutlm-base-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt'''
        ),
        '''microsoft/layoutlm-large-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''microsoft/layoutlm-base-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json'''
        ),
        '''microsoft/layoutlm-large-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json'''
        ),
    },
}

_lowerCamelCase							:  Dict     =				{
    '''microsoft/layoutlm-base-uncased''': 5_1_2,
    '''microsoft/layoutlm-large-uncased''': 5_1_2,
}

_lowerCamelCase							:  List[Any]     =				{
    '''microsoft/layoutlm-base-uncased''': {'''do_lower_case''': True},
    '''microsoft/layoutlm-large-uncased''': {'''do_lower_case''': True},
}





class    __UpperCAmelCase	(			lowerCamelCase__		):
   UpperCamelCase  	=	VOCAB_FILES_NAMES
   UpperCamelCase  	=	PRETRAINED_VOCAB_FILES_MAP
   UpperCamelCase  	=	PRETRAINED_INIT_CONFIGURATION
   UpperCamelCase  	=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   UpperCamelCase  	=	LayoutLMTokenizer



   def __init__(				self				:	Any,						__A				:	Dict=None,						__A				:	Dict=None,						__A				:	int=True,						__A				:	Dict="[UNK]",						__A				:	Union[str, Any]="[SEP]",						__A				:	Union[str, Any]="[PAD]",						__A				:	Optional[Any]="[CLS]",						__A				:	List[Any]="[MASK]",						__A				:	Union[str, Any]=True,						__A				:	Union[str, Any]=None,						**__A				:	int,						):
     super().__init__(
         UpperCAmelCase_,						tokenizer_file=UpperCAmelCase_,						do_lower_case=UpperCAmelCase_,						unk_token=UpperCAmelCase_,						sep_token=UpperCAmelCase_,						pad_token=UpperCAmelCase_,						cls_token=UpperCAmelCase_,						mask_token=UpperCAmelCase_,						tokenize_chinese_chars=UpperCAmelCase_,						strip_accents=UpperCAmelCase_,						**UpperCAmelCase_,						)

     UpperCAmelCase			:				str										=					json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
     if (
         normalizer_state.get('''lowercase''',						UpperCAmelCase_		) != do_lower_case
         or normalizer_state.get('''strip_accents''',						UpperCAmelCase_		) != strip_accents
         or normalizer_state.get('''handle_chinese_chars''',						UpperCAmelCase_		) != tokenize_chinese_chars
     ):
       UpperCAmelCase			:				Optional[int]										=					getattr(UpperCAmelCase_,						normalizer_state.pop('''type'''		)		)
       UpperCAmelCase			:				Dict										=					do_lower_case
       UpperCAmelCase			:				List[str]										=					strip_accents
       UpperCAmelCase			:				Any										=					tokenize_chinese_chars
       UpperCAmelCase			:				Any										=					normalizer_class(**UpperCAmelCase_		)

     UpperCAmelCase			:				int										=					do_lower_case



   def 			__magic_name__       (				self				:	Optional[int],						__A				:	int,						__A				:	List[str]=None		):
     UpperCAmelCase			:				int										=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

     return output



   def 			__magic_name__       (				self				:	Optional[int],						__A				:	Optional[int],						__A				:	Dict = None		):
     UpperCAmelCase			:				Optional[int]										=					[self.sep_token_id]
     UpperCAmelCase			:				Tuple										=					[self.cls_token_id]
     if token_ids_a is None:
       return len(cls + token_ids_a + sep		) * [0]
     return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]





   def 			__magic_name__       (				self				:	Dict,						__A				:	Dict,						__A				:	Optional[Any] = None		):
     UpperCAmelCase			:				Optional[int]										=					self._tokenizer.model.save(UpperCAmelCase_,						name=UpperCAmelCase_		)
     return tuple(UpperCAmelCase_		)












from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


UpperCAmelCase__    :      int       					={
    '''configuration_flaubert''': ['''FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''FlaubertConfig''', '''FlaubertOnnxConfig'''],
    '''tokenization_flaubert''': ['''FlaubertTokenizer'''],
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							UpperCAmelCase__    :      int       					=[
							    '''FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
							    '''FlaubertForMultipleChoice''',
							    '''FlaubertForQuestionAnswering''',
							    '''FlaubertForQuestionAnsweringSimple''',
							    '''FlaubertForSequenceClassification''',
							    '''FlaubertForTokenClassification''',
							    '''FlaubertModel''',
							    '''FlaubertWithLMHeadModel''',
							    '''FlaubertPreTrainedModel''',
							]

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							UpperCAmelCase__    :      List[str]       					=[
							    '''TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
							    '''TFFlaubertForMultipleChoice''',
							    '''TFFlaubertForQuestionAnsweringSimple''',
							    '''TFFlaubertForSequenceClassification''',
							    '''TFFlaubertForTokenClassification''',
							    '''TFFlaubertModel''',
							    '''TFFlaubertPreTrainedModel''',
							    '''TFFlaubertWithLMHeadModel''',
							]


if TYPE_CHECKING:
							from .configuration_flaubert import FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, FlaubertConfig, FlaubertOnnxConfig
							from .tokenization_flaubert import FlaubertTokenizer

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_flaubert import (
														    FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
														    FlaubertForMultipleChoice,
														    FlaubertForQuestionAnswering,
														    FlaubertForQuestionAnsweringSimple,
														    FlaubertForSequenceClassification,
														    FlaubertForTokenClassification,
														    FlaubertModel,
														    FlaubertPreTrainedModel,
														    FlaubertWithLMHeadModel,
														)

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_flaubert import (
														    TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFFlaubertForMultipleChoice,
														    TFFlaubertForQuestionAnsweringSimple,
														    TFFlaubertForSequenceClassification,
														    TFFlaubertForTokenClassification,
														    TFFlaubertModel,
														    TFFlaubertPreTrainedModel,
														    TFFlaubertWithLMHeadModel,
														)

else:
							import sys

							UpperCAmelCase__    :      List[Any]       					=_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





