




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


UpperCAmelCase__    :							int       		=     logging.get_logger(__name__)

UpperCAmelCase__    :							Any       		=     {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase__    :							List[str]       		=     {
    'vocab_file': {
        'microsoft/layoutlm-base-uncased': (
            'https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt'
        ),
        'microsoft/layoutlm-large-uncased': (
            'https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'microsoft/layoutlm-base-uncased': (
            'https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json'
        ),
        'microsoft/layoutlm-large-uncased': (
            'https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json'
        ),
    },
}

UpperCAmelCase__    :							int       		=     {
    'microsoft/layoutlm-base-uncased': 512,
    'microsoft/layoutlm-large-uncased': 512,
}

UpperCAmelCase__    :							Dict       		=     {
    'microsoft/layoutlm-base-uncased': {'do_lower_case': True},
    'microsoft/layoutlm-large-uncased': {'do_lower_case': True},
}

class   UpperCAmelCase    (      A ):
   __SCREAMING_SNAKE_CASE		:				int      					=       VOCAB_FILES_NAMES
   __SCREAMING_SNAKE_CASE		:				Dict      					=       PRETRAINED_VOCAB_FILES_MAP
   __SCREAMING_SNAKE_CASE		:				Tuple      					=       PRETRAINED_INIT_CONFIGURATION
   __SCREAMING_SNAKE_CASE		:				str      					=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __SCREAMING_SNAKE_CASE		:				Tuple      					=       LayoutLMTokenizer






   def __init__(						self :							List[Any]    ,       UpperCamelCase_ :							int=None    ,       UpperCamelCase_ :							Tuple=None    ,       UpperCamelCase_ :							Dict=True    ,       UpperCamelCase_ :							Optional[int]="[UNK]"    ,       UpperCamelCase_ :							Optional[int]="[SEP]"    ,       UpperCamelCase_ :							Any="[PAD]"    ,       UpperCamelCase_ :							Union[str, Any]="[CLS]"    ,       UpperCamelCase_ :							Union[str, Any]="[MASK]"    ,       UpperCamelCase_ :							List[Any]=True    ,       UpperCamelCase_ :							Any=None    ,       **UpperCamelCase_ :							Union[str, Any]    ,       )->      Optional[Any]:
     super().__init__(
         UpperCamelCase_    ,       tokenizer_file=UpperCamelCase_    ,       do_lower_case=UpperCamelCase_    ,       unk_token=UpperCamelCase_    ,       sep_token=UpperCamelCase_    ,       pad_token=UpperCamelCase_    ,       cls_token=UpperCamelCase_    ,       mask_token=UpperCamelCase_    ,       tokenize_chinese_chars=UpperCamelCase_    ,       strip_accents=UpperCamelCase_    ,       **UpperCamelCase_    ,       )

     _lowercase:			str      =     json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
     if (
         normalizer_state.get('lowercase'    ,       UpperCamelCase_  ) != do_lower_case
         or normalizer_state.get('strip_accents'    ,       UpperCamelCase_  ) != strip_accents
         or normalizer_state.get('handle_chinese_chars'    ,       UpperCamelCase_  ) != tokenize_chinese_chars
     ):
       _lowercase:			Tuple      =     getattr(UpperCamelCase_    ,       normalizer_state.pop('type'  )  )
       _lowercase:			List[Any]      =     do_lower_case
       _lowercase:			Any      =     strip_accents
       _lowercase:			Union[str, Any]      =     tokenize_chinese_chars
       _lowercase:			Tuple      =     normalizer_class(**UpperCamelCase_  )

     _lowercase:			Any      =     do_lower_case






   def 			_a    (						self :							Optional[int]    ,       UpperCamelCase_ :							List[str]    ,       UpperCamelCase_ :							Union[str, Any]=None  )->      Optional[int]:
     _lowercase:			Optional[Any]      =     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

     return output






   def 			_a    (						self :							str    ,       UpperCamelCase_ :							List[int]    ,       UpperCamelCase_ :							Optional[List[int]] = None  )->      List[int]:
     _lowercase:			Optional[Any]      =     [self.sep_token_id]
     _lowercase:			Optional[int]      =     [self.cls_token_id]
     if token_ids_a is None:
       return len(cls + token_ids_a + sep  ) * [0]
     return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]






   def 			_a    (						self :							List[Any]    ,       UpperCamelCase_ :							str    ,       UpperCamelCase_ :							Optional[str] = None  )->      Tuple[str]:
     _lowercase:			List[Any]      =     self._tokenizer.model.save(UpperCamelCase_    ,       name=UpperCamelCase_  )
     return tuple(UpperCamelCase_  )






from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available, is_torch_available


UpperCAmelCase__    :							List[str]       		=     {
    'configuration_ernie_m': ['ERNIE_M_PRETRAINED_CONFIG_ARCHIVE_MAP', 'ErnieMConfig'],
}

try:
   if not is_sentencepiece_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase__    :							Any       		=     ['ErnieMTokenizer']

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase__    :							Dict       		=     [
       'ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST',
       'ErnieMForMultipleChoice',
       'ErnieMForQuestionAnswering',
       'ErnieMForSequenceClassification',
       'ErnieMForTokenClassification',
       'ErnieMModel',
       'ErnieMPreTrainedModel',
       'ErnieMForInformationExtraction',
   ]


if TYPE_CHECKING:
   from .configuration_ernie_m import ERNIE_M_PRETRAINED_CONFIG_ARCHIVE_MAP, ErnieMConfig

   try:
      if not is_sentencepiece_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_ernie_m import ErnieMTokenizer

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_ernie_m import (
          ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST,
          ErnieMForInformationExtraction,
          ErnieMForMultipleChoice,
          ErnieMForQuestionAnswering,
          ErnieMForSequenceClassification,
          ErnieMForTokenClassification,
          ErnieMModel,
          ErnieMPreTrainedModel,
      )


else:
   import sys

   UpperCAmelCase__    :							List[Any]       		=     _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)
