"""simple docstring"""







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


SCREAMING_SNAKE_CASE						=    logging.get_logger(__name__)

SCREAMING_SNAKE_CASE						=    {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE						=    {
    'vocab_file': {
        'microsoft/layoutlm-base-uncased': (
            'https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt'
        ),
        'microsoft/layoutlm-large-uncased': (
            'https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'microsoft/layoutlm-base-uncased': (
            'https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json'
        ),
        'microsoft/layoutlm-large-uncased': (
            'https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json'
        ),
    },
}

SCREAMING_SNAKE_CASE						=    {
    'microsoft/layoutlm-base-uncased': 512,
    'microsoft/layoutlm-large-uncased': 512,
}

SCREAMING_SNAKE_CASE						=    {
    'microsoft/layoutlm-base-uncased': {'do_lower_case': True},
    'microsoft/layoutlm-large-uncased': {'do_lower_case': True},
}



class 			_lowerCamelCase      (__lowerCamelCase  ):
				_snake_case  =	VOCAB_FILES_NAMES
				_snake_case  =	PRETRAINED_VOCAB_FILES_MAP
				_snake_case  =	PRETRAINED_INIT_CONFIGURATION
				_snake_case  =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				_snake_case  =	LayoutLMTokenizer







				def __init__(			self     :							Dict      ,       lowerCamelCase_     :							int=None      ,       lowerCamelCase_     :							Union[str, Any]=None      ,       lowerCamelCase_     :							str=True      ,       lowerCamelCase_     :							Any="[UNK]"      ,       lowerCamelCase_     :							Any="[SEP]"      ,       lowerCamelCase_     :							int="[PAD]"      ,       lowerCamelCase_     :							int="[CLS]"      ,       lowerCamelCase_     :							Tuple="[MASK]"      ,       lowerCamelCase_     :							Union[str, Any]=True      ,       lowerCamelCase_     :							Optional[int]=None      ,       **lowerCamelCase_     :							str      ,       ):

					"""simple docstring"""






					super().__init__(
					    lowerCamelCase_      ,       tokenizer_file=lowerCamelCase_      ,       do_lower_case=lowerCamelCase_      ,       unk_token=lowerCamelCase_      ,       sep_token=lowerCamelCase_      ,       pad_token=lowerCamelCase_      ,       cls_token=lowerCamelCase_      ,       mask_token=lowerCamelCase_      ,       tokenize_chinese_chars=lowerCamelCase_      ,       strip_accents=lowerCamelCase_      ,       **lowerCamelCase_      ,       )

					_lowercase			:     Dict      					=				json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
					if (
					    normalizer_state.get('lowercase'      ,       lowerCamelCase_    ) != do_lower_case
					    or normalizer_state.get('strip_accents'      ,       lowerCamelCase_    ) != strip_accents
					    or normalizer_state.get('handle_chinese_chars'      ,       lowerCamelCase_    ) != tokenize_chinese_chars
					):
						_lowercase			:     str      					=				getattr(lowerCamelCase_      ,       normalizer_state.pop('type'    )    )
						_lowercase			:     Any      					=				do_lower_case
						_lowercase			:     List[Any]      					=				strip_accents
						_lowercase			:     Dict      					=				tokenize_chinese_chars
						_lowercase			:     str      					=				normalizer_class(**lowerCamelCase_    )

					_lowercase			:     int      					=				do_lower_case







				def 				__UpperCAmelCase       (			self     :							Dict      ,       lowerCamelCase_     :							int      ,       lowerCamelCase_     :							List[str]=None    ):

					"""simple docstring"""






					_lowercase			:     Any      					=				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
						output += token_ids_a + [self.sep_token_id]

					return output







				def 				__UpperCAmelCase       (			self     :							List[str]      ,       lowerCamelCase_     :							List[int]      ,       lowerCamelCase_     :							Optional[List[int]] = None    ):

					"""simple docstring"""






					_lowercase			:     Union[str, Any]      					=				[self.sep_token_id]
					_lowercase			:     Any      					=				[self.cls_token_id]
					if token_ids_a is None:
						return len(cls + token_ids_a + sep    ) * [0]
					return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]







				def 				__UpperCAmelCase       (			self     :							Any      ,       lowerCamelCase_     :							str      ,       lowerCamelCase_     :							Optional[str] = None    ):

					"""simple docstring"""






					_lowercase			:     Any      					=				self._tokenizer.model.save(lowerCamelCase_      ,       name=lowerCamelCase_    )
					return tuple(lowerCamelCase_    )




"""simple docstring"""







from __future__ import annotations

import re


def   __lowerCAmelCase(		__UpperCAmelCase      ):






	"""simple docstring"""





	def alphanum_key(__UpperCAmelCase      ):
		return [int(__UpperCAmelCase      ) if s.isdigit() else s.lower() for s in re.split('([0-9]+)'  ,__UpperCAmelCase      )]

	return sorted(__UpperCAmelCase  ,key=__UpperCAmelCase      )


if __name__ == "__main__":
						import doctest

						doctest.testmod()



