



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


UpperCAmelCase :				Optional[int]							=logging.get_logger(__name__)

UpperCAmelCase :				List[str]							={"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase :				str							={
    """vocab_file""": {
        """microsoft/layoutlm-base-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt"""
        ),
        """microsoft/layoutlm-large-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """microsoft/layoutlm-base-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json"""
        ),
        """microsoft/layoutlm-large-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json"""
        ),
    },
}

UpperCAmelCase :				str							={
    """microsoft/layoutlm-base-uncased""": 512,
    """microsoft/layoutlm-large-uncased""": 512,
}

UpperCAmelCase :				Any							={
    """microsoft/layoutlm-base-uncased""": {"""do_lower_case""": True},
    """microsoft/layoutlm-large-uncased""": {"""do_lower_case""": True},
}


class    _lowercase (a_   ):
				'''simple docstring'''







				lowercase__     =    VOCAB_FILES_NAMES
				lowercase__     =    PRETRAINED_VOCAB_FILES_MAP
				lowercase__     =    PRETRAINED_INIT_CONFIGURATION
				lowercase__     =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowercase__     =    LayoutLMTokenizer

				def __init__(			self  ,			snake_case__=None  ,			snake_case__=None  ,			snake_case__=True  ,			snake_case__="[UNK]"  ,			snake_case__="[SEP]"  ,			snake_case__="[PAD]"  ,			snake_case__="[CLS]"  ,			snake_case__="[MASK]"  ,			snake_case__=True  ,			snake_case__=None  ,			**snake_case__  ,			):

									'''simple docstring'''


									super().__init__(
									    snake_case__  ,			tokenizer_file=snake_case__  ,			do_lower_case=snake_case__  ,			unk_token=snake_case__  ,			sep_token=snake_case__  ,			pad_token=snake_case__  ,			cls_token=snake_case__  ,			mask_token=snake_case__  ,			tokenize_chinese_chars=snake_case__  ,			strip_accents=snake_case__  ,			**snake_case__  ,			)

									UpperCamelCase_			= json.loads(self.backend_tokenizer.normalizer.__getstate__()     )
									if (
									    normalizer_state.get("lowercase"  ,			snake_case__     ) != do_lower_case
									    or normalizer_state.get("strip_accents"  ,			snake_case__     ) != strip_accents
									    or normalizer_state.get("handle_chinese_chars"  ,			snake_case__     ) != tokenize_chinese_chars
									):
														UpperCamelCase_			= getattr(snake_case__  ,			normalizer_state.pop("type"     )     )
														UpperCamelCase_			= do_lower_case
														UpperCamelCase_			= strip_accents
														UpperCamelCase_			= tokenize_chinese_chars
														UpperCamelCase_			= normalizer_class(**snake_case__     )

									UpperCamelCase_			= do_lower_case

				def    _lowerCamelCase       (			self  ,			snake_case__  ,			snake_case__=None     ):

									'''simple docstring'''


									UpperCamelCase_			= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
														output += token_ids_a + [self.sep_token_id]

									return output

				def    _lowerCamelCase       (			self  ,			snake_case__  ,			snake_case__ = None     ):

									'''simple docstring'''


									UpperCamelCase_			= [self.sep_token_id]
									UpperCamelCase_			= [self.cls_token_id]
									if token_ids_a is None:
														return len(cls + token_ids_a + sep     ) * [0]
									return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]

				def    _lowerCamelCase       (			self  ,			snake_case__  ,			snake_case__ = None     ):

									'''simple docstring'''


									UpperCamelCase_			= self._tokenizer.model.save(snake_case__  ,			name=snake_case__     )
									return tuple(snake_case__     )







from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


UpperCAmelCase_       : List[Any]							=    {
    """configuration_retribert""": ["""RETRIBERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """RetriBertConfig"""],
    """tokenization_retribert""": ["""RetriBertTokenizer"""],
}

try:
      if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      UpperCAmelCase_       : str							=    ["""RetriBertTokenizerFast"""]

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      UpperCAmelCase_       : Optional[Any]							=    [
          """RETRIBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """RetriBertModel""",
          """RetriBertPreTrainedModel""",
      ]


if TYPE_CHECKING:
      from .configuration_retribert import RETRIBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, RetriBertConfig
      from .tokenization_retribert import RetriBertTokenizer

      try:
            if not is_tokenizers_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .tokenization_retribert_fast import RetriBertTokenizerFast

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_retribert import (
                RETRIBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
                RetriBertModel,
                RetriBertPreTrainedModel,
            )

else:
      import sys

      UpperCAmelCase_       : Tuple							=    _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
