


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


lowerCamelCase		=						logging.get_logger(__name__)

lowerCamelCase		=						{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase		=						{
    '''vocab_file''': {
        '''microsoft/layoutlm-base-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt'''
        ),
        '''microsoft/layoutlm-large-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''microsoft/layoutlm-base-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json'''
        ),
        '''microsoft/layoutlm-large-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json'''
        ),
    },
}

lowerCamelCase		=						{
    '''microsoft/layoutlm-base-uncased''': 512,
    '''microsoft/layoutlm-large-uncased''': 512,
}

lowerCamelCase		=						{
    '''microsoft/layoutlm-base-uncased''': {'''do_lower_case''': True},
    '''microsoft/layoutlm-large-uncased''': {'''do_lower_case''': True},
}





class    _a				(							snake_case__):
	_a    :    Optional[Any]        =				VOCAB_FILES_NAMES
	_a    :    Optional[int]        =				PRETRAINED_VOCAB_FILES_MAP
	_a    :    Tuple        =				PRETRAINED_INIT_CONFIGURATION
	_a    :    Union[str, Any]        =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_a    :    List[str]        =				LayoutLMTokenizer




	def __init__(   self		:						Optional[Any]						,   _SCREAMING_SNAKE_CASE		:						Optional[Any]=None						,   _SCREAMING_SNAKE_CASE		:						Union[str, Any]=None						,   _SCREAMING_SNAKE_CASE		:						Any=True						,   _SCREAMING_SNAKE_CASE		:						List[str]="[UNK]"						,   _SCREAMING_SNAKE_CASE		:						str="[SEP]"						,   _SCREAMING_SNAKE_CASE		:						Tuple="[PAD]"						,   _SCREAMING_SNAKE_CASE		:						Dict="[CLS]"						,   _SCREAMING_SNAKE_CASE		:						Optional[Any]="[MASK]"						,   _SCREAMING_SNAKE_CASE		:						int=True						,   _SCREAMING_SNAKE_CASE		:						Any=None						,   **_SCREAMING_SNAKE_CASE		:						int						,   )->			Dict:
			super().__init__(
			    _A						,   tokenizer_file=_A						,   do_lower_case=_A						,   unk_token=_A						,   sep_token=_A						,   pad_token=_A						,   cls_token=_A						,   mask_token=_A						,   tokenize_chinese_chars=_A						,   strip_accents=_A						,   **_A						,   )

			lowerCAmelCase__  :     List[Any]       	=   json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
			if (
			    normalizer_state.get('''lowercase'''						,   _A						) != do_lower_case
			    or normalizer_state.get('''strip_accents'''						,   _A						) != strip_accents
			    or normalizer_state.get('''handle_chinese_chars'''						,   _A						) != tokenize_chinese_chars
			):
					lowerCAmelCase__  :     Dict       	=   getattr(_A						,   normalizer_state.pop('''type'''						)						)
					lowerCAmelCase__  :     int       	=   do_lower_case
					lowerCAmelCase__  :     Tuple       	=   strip_accents
					lowerCAmelCase__  :     Tuple       	=   tokenize_chinese_chars
					lowerCAmelCase__  :     int       	=   normalizer_class(**_A						)

			lowerCAmelCase__  :     Optional[Any]       	=   do_lower_case




	def 				UpperCAmelCase__(   self		:						Any						,   _SCREAMING_SNAKE_CASE		:						str						,   _SCREAMING_SNAKE_CASE		:						List[str]=None						)->			List[Any]:
			lowerCAmelCase__  :     Any       	=   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

			if token_ids_a:
					output += token_ids_a + [self.sep_token_id]

			return output




	def 				UpperCAmelCase__(   self		:						List[Any]						,   _SCREAMING_SNAKE_CASE		:						Any						,   _SCREAMING_SNAKE_CASE		:						str = None						)->			Optional[int]:
			lowerCAmelCase__  :     Optional[int]       	=   [self.sep_token_id]
			lowerCAmelCase__  :     str       	=   [self.cls_token_id]
			if token_ids_a is None:
					return len(cls + token_ids_a + sep						) * [0]
			return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]




	def 				UpperCAmelCase__(   self		:						Optional[int]						,   _SCREAMING_SNAKE_CASE		:						Tuple						,   _SCREAMING_SNAKE_CASE		:						int = None						)->			List[str]:
			lowerCAmelCase__  :     str       	=   self._tokenizer.model.save(_A						,   name=_A						)
			return tuple(_A						)










from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCamelCase__            =     {
    """configuration_marian""": ["""MARIAN_PRETRAINED_CONFIG_ARCHIVE_MAP""", """MarianConfig""", """MarianOnnxConfig"""],
}

try:
					if not is_sentencepiece_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase__            =     ["""MarianTokenizer"""]

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase__            =     [
					    """MARIAN_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """MarianForCausalLM""",
					    """MarianModel""",
					    """MarianMTModel""",
					    """MarianPreTrainedModel""",
					]

try:
					if not is_tf_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase__            =     ["""TFMarianModel""", """TFMarianMTModel""", """TFMarianPreTrainedModel"""]

try:
					if not is_flax_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase__            =     ["""FlaxMarianModel""", """FlaxMarianMTModel""", """FlaxMarianPreTrainedModel"""]

if TYPE_CHECKING:
					from .configuration_marian import MARIAN_PRETRAINED_CONFIG_ARCHIVE_MAP, MarianConfig, MarianOnnxConfig

					try:
										if not is_sentencepiece_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .tokenization_marian import MarianTokenizer

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_marian import (
										    MARIAN_PRETRAINED_MODEL_ARCHIVE_LIST,
										    MarianForCausalLM,
										    MarianModel,
										    MarianMTModel,
										    MarianPreTrainedModel,
										)

					try:
										if not is_tf_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_tf_marian import TFMarianModel, TFMarianMTModel, TFMarianPreTrainedModel

					try:
										if not is_flax_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_flax_marian import FlaxMarianModel, FlaxMarianMTModel, FlaxMarianPreTrainedModel

else:
					import sys

					UpperCamelCase__            =     _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




