



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


UpperCAmelCase :				Optional[int]							=logging.get_logger(__name__)

UpperCAmelCase :				List[str]							={"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase :				str							={
    """vocab_file""": {
        """microsoft/layoutlm-base-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt"""
        ),
        """microsoft/layoutlm-large-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """microsoft/layoutlm-base-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json"""
        ),
        """microsoft/layoutlm-large-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json"""
        ),
    },
}

UpperCAmelCase :				str							={
    """microsoft/layoutlm-base-uncased""": 512,
    """microsoft/layoutlm-large-uncased""": 512,
}

UpperCAmelCase :				Any							={
    """microsoft/layoutlm-base-uncased""": {"""do_lower_case""": True},
    """microsoft/layoutlm-large-uncased""": {"""do_lower_case""": True},
}


class    _lowercase (a_   ):
				'''simple docstring'''







				lowercase__     =    VOCAB_FILES_NAMES
				lowercase__     =    PRETRAINED_VOCAB_FILES_MAP
				lowercase__     =    PRETRAINED_INIT_CONFIGURATION
				lowercase__     =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowercase__     =    LayoutLMTokenizer

				def __init__(			self  ,			snake_case__=None  ,			snake_case__=None  ,			snake_case__=True  ,			snake_case__="[UNK]"  ,			snake_case__="[SEP]"  ,			snake_case__="[PAD]"  ,			snake_case__="[CLS]"  ,			snake_case__="[MASK]"  ,			snake_case__=True  ,			snake_case__=None  ,			**snake_case__  ,			):

									'''simple docstring'''


									super().__init__(
									    snake_case__  ,			tokenizer_file=snake_case__  ,			do_lower_case=snake_case__  ,			unk_token=snake_case__  ,			sep_token=snake_case__  ,			pad_token=snake_case__  ,			cls_token=snake_case__  ,			mask_token=snake_case__  ,			tokenize_chinese_chars=snake_case__  ,			strip_accents=snake_case__  ,			**snake_case__  ,			)

									UpperCamelCase_			= json.loads(self.backend_tokenizer.normalizer.__getstate__()     )
									if (
									    normalizer_state.get("lowercase"  ,			snake_case__     ) != do_lower_case
									    or normalizer_state.get("strip_accents"  ,			snake_case__     ) != strip_accents
									    or normalizer_state.get("handle_chinese_chars"  ,			snake_case__     ) != tokenize_chinese_chars
									):
														UpperCamelCase_			= getattr(snake_case__  ,			normalizer_state.pop("type"     )     )
														UpperCamelCase_			= do_lower_case
														UpperCamelCase_			= strip_accents
														UpperCamelCase_			= tokenize_chinese_chars
														UpperCamelCase_			= normalizer_class(**snake_case__     )

									UpperCamelCase_			= do_lower_case

				def    _lowerCamelCase       (			self  ,			snake_case__  ,			snake_case__=None     ):

									'''simple docstring'''


									UpperCamelCase_			= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
														output += token_ids_a + [self.sep_token_id]

									return output

				def    _lowerCamelCase       (			self  ,			snake_case__  ,			snake_case__ = None     ):

									'''simple docstring'''


									UpperCamelCase_			= [self.sep_token_id]
									UpperCamelCase_			= [self.cls_token_id]
									if token_ids_a is None:
														return len(cls + token_ids_a + sep     ) * [0]
									return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]

				def    _lowerCamelCase       (			self  ,			snake_case__  ,			snake_case__ = None     ):

									'''simple docstring'''


									UpperCamelCase_			= self._tokenizer.model.save(snake_case__  ,			name=snake_case__     )
									return tuple(snake_case__     )






'''simple docstring'''







import argparse
import os
import re


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_dummies.py
_lowerCamelCase				:		str				      =    'src/transformers'

# Matches is_xxx_available()
_lowerCamelCase				:		str				      =    re.compile(R'is\_([a-z_]*)_available()')
# Matches from xxx import bla
_lowerCamelCase				:		Union[str, Any]				      =    re.compile(R'\s+from\s+\S*\s+import\s+([^\(\s].*)\n')
_lowerCamelCase				:		Dict				      =    re.compile(R'^\s+if\s+not\s+\(?is\_[a-z_]*\_available\(\)')


_lowerCamelCase				:		int				      =    '\n{0} = None\n'

_lowerCamelCase				:		List[Any]				      =    '\nclass {0}(metaclass=DummyObject):\n    _backends = {1}\n\n    def __init__(self, *args, **kwargs):\n        requires_backends(self, {1})\n'


_lowerCamelCase				:		Any				      =    '\ndef {0}(*args, **kwargs):\n    requires_backends({0}, {1})\n'






def 						_lowercase						(      lowercase_    ):



		'''simple docstring'''







		if _re_test_backend.search(lowercase_    ) is None:
				return None
		_snake_case      			=			[b[0] for b in _re_backend.findall(lowercase_    )]
		backends.sort()
		return "_and_".join(lowercase_    )






def 						_lowercase						(      ):



		'''simple docstring'''







		with open(os.path.join(lowercase_		,				"__init__.py"    )		,				"r"		,				encoding="utf-8"		,				newline="\n"    ) as f:
				_snake_case      			=			f.readlines()

		# Get to the point we do the actual imports for type checking
		_snake_case      			=			0
		while not lines[line_index].startswith("if TYPE_CHECKING"    ):
				line_index += 1

		_snake_case      			=			{}
		# Go through the end of the file
		while line_index < len(lowercase_    ):
				# If the line is an if is_backend_available, we grab all objects associated.
				_snake_case      			=			find_backend(lines[line_index]    )
				if backend is not None:
						while not lines[line_index].startswith("    else:"    ):
								line_index += 1
						line_index += 1

						_snake_case      			=			[]
						# Until we unindent, add backend objects to the list
						while len(lines[line_index]    ) <= 1 or lines[line_index].startswith(" " * 8    ):
								_snake_case      			=			lines[line_index]
								_snake_case      			=			_re_single_line_import.search(lowercase_    )
								if single_line_import_search is not None:
										objects.extend(single_line_import_search.groups()[0].split(", "    )    )
								elif line.startswith(" " * 12    ):
										objects.append(line[12:-2]    )
								line_index += 1

						_snake_case      			=			objects
				else:
						line_index += 1

		return backend_specific_objects






def 						_lowercase						(      lowercase_		,				lowercase_    ):



		'''simple docstring'''







		if name.isupper():
				return DUMMY_CONSTANT.format(lowercase_    )
		elif name.islower():
				return DUMMY_FUNCTION.format(lowercase_		,				lowercase_    )
		else:
				return DUMMY_CLASS.format(lowercase_		,				lowercase_    )






def 						_lowercase						(      lowercase_=None    ):



		'''simple docstring'''







		if backend_specific_objects is None:
				_snake_case      			=			read_init()
		# For special correspondence backend to module name as used in the function requires_modulename
		_snake_case      			=			{}

		for backend, objects in backend_specific_objects.items():
				_snake_case      			=			"[" + ", ".join(f'''"{b}"''' for b in backend.split("_and_"    )    ) + "]"
				_snake_case      			=			"# This file is autogenerated by the command `make fix-copies`, do not edit.\n"
				dummy_file += "from ..utils import DummyObject, requires_backends\n\n"
				dummy_file += "\n".join([create_dummy_object(lowercase_		,				lowercase_    ) for o in objects]    )
				_snake_case      			=			dummy_file

		return dummy_files






def 						_lowercase						(      lowercase_=False    ):



		'''simple docstring'''







		_snake_case      			=			create_dummy_files()
		# For special correspondence backend to shortcut as used in utils/dummy_xxx_objects.py
		_snake_case      			=			{"torch": "pt"}

		# Locate actual dummy modules and read their content.
		_snake_case      			=			os.path.join(lowercase_		,				"utils"    )
		_snake_case      			=			{
		    backend: os.path.join(lowercase_		,				f'''dummy_{short_names.get(lowercase_		,				lowercase_    )}_objects.py'''    )
		    for backend in dummy_files.keys()
		}

		_snake_case      			=			{}
		for backend, file_path in dummy_file_paths.items():
				if os.path.isfile(lowercase_    ):
						with open(lowercase_		,				"r"		,				encoding="utf-8"		,				newline="\n"    ) as f:
								_snake_case      			=			f.read()
				else:
						_snake_case      			=			""

		for backend in dummy_files.keys():
				if dummy_files[backend] != actual_dummies[backend]:
						if overwrite:
								print(
								    f'''Updating transformers.utils.dummy_{short_names.get(lowercase_		,				lowercase_    )}_objects.py as the main '''
								    "__init__ has new objects."    )
								with open(dummy_file_paths[backend]		,				"w"		,				encoding="utf-8"		,				newline="\n"    ) as f:
										f.write(dummy_files[backend]    )
						else:
								raise ValueError(
								    "The main __init__ has objects that are not present in "
								    f'''transformers.utils.dummy_{short_names.get(lowercase_		,				lowercase_    )}_objects.py. Run `make fix-copies` '''
								    "to fix this."    )


if __name__ == "__main__":
						_lowerCamelCase				:		int				      =    argparse.ArgumentParser()
						parser.add_argument('--fix_and_overwrite', action='store_true', help='Whether to fix inconsistencies.')
						_lowerCamelCase				:		List[Any]				      =    parser.parse_args()

						check_dummies(args.fix_and_overwrite)





