
'''simple docstring'''





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


a__		: List[Any]  							=							logging.get_logger(__name__)

a__		: List[str]  							=							{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

a__		: Optional[int]  							=							{
    """vocab_file""": {
        """microsoft/layoutlm-base-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt"""
        ),
        """microsoft/layoutlm-large-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """microsoft/layoutlm-base-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json"""
        ),
        """microsoft/layoutlm-large-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json"""
        ),
    },
}

a__		: Tuple  							=							{
    """microsoft/layoutlm-base-uncased""": 5_1_2,
    """microsoft/layoutlm-large-uncased""": 5_1_2,
}

a__		: List[Any]  							=							{
    """microsoft/layoutlm-base-uncased""": {"""do_lower_case""": True},
    """microsoft/layoutlm-large-uncased""": {"""do_lower_case""": True},
}


class 			UpperCamelCase__	(     lowerCAmelCase__):
 UpperCAmelCase__     :     List[str]    					=       VOCAB_FILES_NAMES
 UpperCAmelCase__     :     Dict    					=       PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase__     :     Optional[int]    					=       PRETRAINED_INIT_CONFIGURATION
 UpperCAmelCase__     :     Any    					=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase__     :     str    					=       LayoutLMTokenizer






 def __init__(      self	:Optional[int]						,					_A	:Dict=None						,					_A	:int=None						,					_A	:Tuple=True						,					_A	:Optional[int]="[UNK]"						,					_A	:Tuple="[SEP]"						,					_A	:int="[PAD]"						,					_A	:List[Any]="[CLS]"						,					_A	:Union[str, Any]="[MASK]"						,					_A	:Optional[Any]=True						,					_A	:Union[str, Any]=None						,					**_A	:Optional[int]						,					)					->      Tuple:







       '''simple docstring'''



       super().__init__(
           _A						,					tokenizer_file=_A						,					do_lower_case=_A						,					unk_token=_A						,					sep_token=_A						,					pad_token=_A						,					cls_token=_A						,					mask_token=_A						,					tokenize_chinese_chars=_A						,					strip_accents=_A						,					**_A						,					)

       __A													=      json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
       if (
           normalizer_state.get('lowercase'						,					_A      ) != do_lower_case
           or normalizer_state.get('strip_accents'						,					_A      ) != strip_accents
           or normalizer_state.get('handle_chinese_chars'						,					_A      ) != tokenize_chinese_chars
       ):
             __A													=      getattr(_A						,					normalizer_state.pop('type'      )      )
             __A													=      do_lower_case
             __A													=      strip_accents
             __A													=      tokenize_chinese_chars
             __A													=      normalizer_class(**_A      )

       __A													=      do_lower_case






 def 						lowercase_    (      self	:Tuple						,					_A	:Optional[int]						,					_A	:str=None      )					->      Dict:







       '''simple docstring'''



       __A													=      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
             output += token_ids_a + [self.sep_token_id]

       return output






 def 						lowercase_    (      self	:List[str]						,					_A	:List[int]						,					_A	:Optional[List[int]] = None      )					->      Tuple:







       '''simple docstring'''



       __A													=      [self.sep_token_id]
       __A													=      [self.cls_token_id]
       if token_ids_a is None:
             return len(cls + token_ids_a + sep      ) * [0]
       return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]






 def 						lowercase_    (      self	:str						,					_A	:str						,					_A	:Optional[str] = None      )					->      Union[str, Any]:







       '''simple docstring'''



       __A													=      self._tokenizer.model.save(_A						,					name=_A      )
       return tuple(_A      )











'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available, is_torch_available


SCREAMING_SNAKE_CASE__     : int				=   {"""configuration_bert_generation""": ["""BertGenerationConfig"""]}

try:
							if not is_sentencepiece_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							SCREAMING_SNAKE_CASE__     : List[Any]				=   ["""BertGenerationTokenizer"""]

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							SCREAMING_SNAKE_CASE__     : Optional[Any]				=   [
							    """BertGenerationDecoder""",
							    """BertGenerationEncoder""",
							    """BertGenerationPreTrainedModel""",
							    """load_tf_weights_in_bert_generation""",
							]


if TYPE_CHECKING:
							from .configuration_bert_generation import BertGenerationConfig

							try:
														if not is_sentencepiece_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_bert_generation import BertGenerationTokenizer

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_bert_generation import (
														    BertGenerationDecoder,
														    BertGenerationEncoder,
														    BertGenerationPreTrainedModel,
														    load_tf_weights_in_bert_generation,
														)

else:
							import sys

							SCREAMING_SNAKE_CASE__     : Any				=   _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
