
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


__lowerCamelCase								=					logging.get_logger(__name__)

__lowerCamelCase								=					{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__lowerCamelCase								=					{
    'vocab_file': {
        'microsoft/layoutlm-base-uncased': (
            'https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt'
        ),
        'microsoft/layoutlm-large-uncased': (
            'https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'microsoft/layoutlm-base-uncased': (
            'https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json'
        ),
        'microsoft/layoutlm-large-uncased': (
            'https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json'
        ),
    },
}

__lowerCamelCase								=					{
    'microsoft/layoutlm-base-uncased': 5_1_2,
    'microsoft/layoutlm-large-uncased': 5_1_2,
}

__lowerCamelCase								=					{
    'microsoft/layoutlm-base-uncased': {'do_lower_case': True},
    'microsoft/layoutlm-large-uncased': {'do_lower_case': True},
}
class    _UpperCamelCase(   SCREAMING_SNAKE_CASE				):
						__A:						Tuple				   =      VOCAB_FILES_NAMES
						__A:						Optional[Any]				   =      PRETRAINED_VOCAB_FILES_MAP
						__A:						Tuple				   =      PRETRAINED_INIT_CONFIGURATION
						__A:						Any				   =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						__A:						int				   =      LayoutLMTokenizer

						def __init__(					self		:			int	,						_lowerCamelCase		:			List[str]=None	,						_lowerCamelCase		:			int=None	,						_lowerCamelCase		:			Tuple=True	,						_lowerCamelCase		:			Union[str, Any]="[UNK]"	,						_lowerCamelCase		:			str="[SEP]"	,						_lowerCamelCase		:			str="[PAD]"	,						_lowerCamelCase		:			Dict="[CLS]"	,						_lowerCamelCase		:			str="[MASK]"	,						_lowerCamelCase		:			int=True	,						_lowerCamelCase		:			int=None	,						**_lowerCamelCase		:			List[Any]	,						):
													super().__init__(
													    _lowerCamelCase	,						tokenizer_file=_lowerCamelCase	,						do_lower_case=_lowerCamelCase	,						unk_token=_lowerCamelCase	,						sep_token=_lowerCamelCase	,						pad_token=_lowerCamelCase	,						cls_token=_lowerCamelCase	,						mask_token=_lowerCamelCase	,						tokenize_chinese_chars=_lowerCamelCase	,						strip_accents=_lowerCamelCase	,						**_lowerCamelCase	,						)

													_UpperCAmelCase  :			Tuple	       =		json.loads(self.backend_tokenizer.normalizer.__getstate__()			)
													if (
													    normalizer_state.get("lowercase"	,						_lowerCamelCase			) != do_lower_case
													    or normalizer_state.get("strip_accents"	,						_lowerCamelCase			) != strip_accents
													    or normalizer_state.get("handle_chinese_chars"	,						_lowerCamelCase			) != tokenize_chinese_chars
													):
																				_UpperCAmelCase  :			Tuple	       =		getattr(_lowerCamelCase	,						normalizer_state.pop("type"			)			)
																				_UpperCAmelCase  :			int	       =		do_lower_case
																				_UpperCAmelCase  :			Union[str, Any]	       =		strip_accents
																				_UpperCAmelCase  :			Dict	       =		tokenize_chinese_chars
																				_UpperCAmelCase  :			Optional[int]	       =		normalizer_class(**_lowerCamelCase			)

													_UpperCAmelCase  :			str	       =		do_lower_case

						def 						a__ (					self		:			Dict	,						_lowerCamelCase		:			Dict	,						_lowerCamelCase		:			Any=None			):
													_UpperCAmelCase  :			Dict	       =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

													if token_ids_a:
																				output += token_ids_a + [self.sep_token_id]

													return output

						def 						a__ (					self		:			int	,						_lowerCamelCase		:			List[int]	,						_lowerCamelCase		:			Optional[List[int]] = None			):
													_UpperCAmelCase  :			Union[str, Any]	       =		[self.sep_token_id]
													_UpperCAmelCase  :			int	       =		[self.cls_token_id]
													if token_ids_a is None:
																				return len(cls + token_ids_a + sep			) * [0]
													return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]






						def 						a__ (					self		:			Tuple	,						_lowerCamelCase		:			str	,						_lowerCamelCase		:			Optional[str] = None			):
													_UpperCAmelCase  :			Tuple	       =		self._tokenizer.model.save(_lowerCamelCase	,						name=_lowerCamelCase			)
													return tuple(_lowerCamelCase			)






import torch

from accelerate import Accelerator, DistributedType
class    _UpperCamelCase:

						def __enter__(					self		:			str			):
													if self.enabled:
																				_UpperCAmelCase  :			int	       =		self.model.no_sync()
																				self.model_sync_obj.__enter__()

													return self

						def __exit__(					self		:			str	,						_lowerCamelCase		:			Union[str, Any]	,						_lowerCamelCase		:			Tuple	,						_lowerCamelCase		:			Optional[Any]			):
													if self.enabled:
																				# Average all models on exit
																				self._sync_and_avg_model_params()
																				self.model_sync_obj.__exit__(_lowerCamelCase	,						_lowerCamelCase	,						_lowerCamelCase			)

						def __init__(					self		:			str	,						_lowerCamelCase		:			Accelerator	,						_lowerCamelCase		:			torch.nn.Module	,						_lowerCamelCase		:			int	,						_lowerCamelCase		:			bool = True			):
													if accelerator.distributed_type not in [
													    DistributedType.NO,
													    DistributedType.MULTI_CPU,
													    DistributedType.MULTI_GPU,
													]:
																				raise NotImplementedError("LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)"			)
													_UpperCAmelCase  :			Any	       =		enabled and accelerator.distributed_type != DistributedType.NO
													_UpperCAmelCase  :			str	       =		0
													if self.enabled:
																				_UpperCAmelCase  :			Tuple	       =		accelerator
																				_UpperCAmelCase  :			List[str]	       =		model
																				_UpperCAmelCase  :			Dict	       =		local_sgd_steps

						def 						a__ (					self		:			Optional[int]			):
													self.num_steps += 1
													if not self.enabled:
																				return

													if self.num_steps % self.local_sgd_steps == 0:
																				self._sync_and_avg_model_params()






						def 						a__ (					self		:			Tuple			):
													self.accelerator.wait_for_everyone()
													with self.accelerator.autocast():
																				for param in self.model.parameters():
																											_UpperCAmelCase  :			Optional[Any]	       =		self.accelerator.reduce(param.data	,						reduction="mean"			)




