




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


UpperCAmelCase_    =logging.get_logger(__name__)

UpperCAmelCase_    ={"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase_    ={
    """vocab_file""": {
        """microsoft/layoutlm-base-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt"""
        ),
        """microsoft/layoutlm-large-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """microsoft/layoutlm-base-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json"""
        ),
        """microsoft/layoutlm-large-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json"""
        ),
    },
}

UpperCAmelCase_    ={
    """microsoft/layoutlm-base-uncased""": 512,
    """microsoft/layoutlm-large-uncased""": 512,
}

UpperCAmelCase_    ={
    """microsoft/layoutlm-base-uncased""": {"""do_lower_case""": True},
    """microsoft/layoutlm-large-uncased""": {"""do_lower_case""": True},
}


class 				__UpperCamelCase      (			__UpperCAmelCase	):





				'''simple docstring'''



				__a				:			Union[str, Any]					    =VOCAB_FILES_NAMES
				__a				:			List[Any]					    =PRETRAINED_VOCAB_FILES_MAP
				__a				:			List[str]					    =PRETRAINED_INIT_CONFIGURATION
				__a				:			Optional[int]					    =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__a				:			Dict					    =LayoutLMTokenizer



				def __init__(			self							,			UpperCAmelCase_=None							,			UpperCAmelCase_=None							,			UpperCAmelCase_=True							,			UpperCAmelCase_="[UNK]"							,			UpperCAmelCase_="[SEP]"							,			UpperCAmelCase_="[PAD]"							,			UpperCAmelCase_="[CLS]"							,			UpperCAmelCase_="[MASK]"							,			UpperCAmelCase_=True							,			UpperCAmelCase_=None							,			**UpperCAmelCase_							,			):
											super().__init__(
											    UpperCAmelCase_							,			tokenizer_file=UpperCAmelCase_							,			do_lower_case=UpperCAmelCase_							,			unk_token=UpperCAmelCase_							,			sep_token=UpperCAmelCase_							,			pad_token=UpperCAmelCase_							,			cls_token=UpperCAmelCase_							,			mask_token=UpperCAmelCase_							,			tokenize_chinese_chars=UpperCAmelCase_							,			strip_accents=UpperCAmelCase_							,			**UpperCAmelCase_							,			)

											lowerCAmelCase =		json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
											if (
											    normalizer_state.get('''lowercase'''							,			UpperCAmelCase_    ) != do_lower_case
											    or normalizer_state.get('''strip_accents'''							,			UpperCAmelCase_    ) != strip_accents
											    or normalizer_state.get('''handle_chinese_chars'''							,			UpperCAmelCase_    ) != tokenize_chinese_chars
											):
																		lowerCAmelCase =		getattr(UpperCAmelCase_							,			normalizer_state.pop('''type'''    )    )
																		lowerCAmelCase =		do_lower_case
																		lowerCAmelCase =		strip_accents
																		lowerCAmelCase =		tokenize_chinese_chars
																		lowerCAmelCase =		normalizer_class(**UpperCAmelCase_    )

											lowerCAmelCase =		do_lower_case



				def    __snake_case		(			self							,			UpperCAmelCase_							,			UpperCAmelCase_=None    ):
											lowerCAmelCase =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

											if token_ids_a:
																		output += token_ids_a + [self.sep_token_id]

											return output



				def    __snake_case		(			self							,			UpperCAmelCase_							,			UpperCAmelCase_ = None    ):
											lowerCAmelCase =		[self.sep_token_id]
											lowerCAmelCase =		[self.cls_token_id]
											if token_ids_a is None:
																		return len(cls + token_ids_a + sep    ) * [0]
											return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]






				def    __snake_case		(			self							,			UpperCAmelCase_							,			UpperCAmelCase_ = None    ):
											lowerCAmelCase =		self._tokenizer.model.save(UpperCAmelCase_							,			name=UpperCAmelCase_    )
											return tuple(UpperCAmelCase_    )









from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


UpperCAmelCase_    ={"""configuration_vilt""": ["""VILT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """ViltConfig"""]}

try:
		if not is_vision_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCAmelCase_    =["""ViltFeatureExtractor"""]
		UpperCAmelCase_    =["""ViltImageProcessor"""]
		UpperCAmelCase_    =["""ViltProcessor"""]

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCAmelCase_    =[
		    """VILT_PRETRAINED_MODEL_ARCHIVE_LIST""",
		    """ViltForImageAndTextRetrieval""",
		    """ViltForImagesAndTextClassification""",
		    """ViltForTokenClassification""",
		    """ViltForMaskedLM""",
		    """ViltForQuestionAnswering""",
		    """ViltLayer""",
		    """ViltModel""",
		    """ViltPreTrainedModel""",
		]


if TYPE_CHECKING:
		from .configuration_vilt import VILT_PRETRAINED_CONFIG_ARCHIVE_MAP, ViltConfig

		try:
				if not is_vision_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .feature_extraction_vilt import ViltFeatureExtractor
				from .image_processing_vilt import ViltImageProcessor
				from .processing_vilt import ViltProcessor

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_vilt import (
				    VILT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    ViltForImageAndTextRetrieval,
				    ViltForImagesAndTextClassification,
				    ViltForMaskedLM,
				    ViltForQuestionAnswering,
				    ViltForTokenClassification,
				    ViltLayer,
				    ViltModel,
				    ViltPreTrainedModel,
				)


else:
		import sys

		UpperCAmelCase_    =_LazyModule(__name__, globals()["""__file__"""], _import_structure)



