





from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase__    :      Any       					=logging.get_logger(__name__)

UpperCAmelCase__    :      Tuple       					={
    '''google/reformer-crime-and-punishment''': (
        '''https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/config.json'''
    ),
    '''google/reformer-enwik8''': '''https://huggingface.co/google/reformer-enwik8/resolve/main/config.json''',
}

class    __A  (    a     ):
				__A  =					"""reformer"""
				__A  =					["""past_buckets_states"""]
				__A  =					{}


				def __init__(		self ,			UpperCAmelCase_=64 ,			UpperCAmelCase_=["local", "lsh", "local", "lsh", "local", "lsh"] ,			UpperCAmelCase_=1.0 ,			UpperCAmelCase_=True ,			UpperCAmelCase_=[64, 64] ,			UpperCAmelCase_=[64, 192] ,			UpperCAmelCase_=0 ,			UpperCAmelCase_=2 ,			UpperCAmelCase_=512 ,			UpperCAmelCase_=None ,			UpperCAmelCase_="relu" ,			UpperCAmelCase_=0.0_5 ,			UpperCAmelCase_=256 ,			UpperCAmelCase_=0.0_2 ,			UpperCAmelCase_=False ,			UpperCAmelCase_=1E-12 ,			UpperCAmelCase_=1 ,			UpperCAmelCase_=0 ,			UpperCAmelCase_=0.0_5 ,			UpperCAmelCase_=64 ,			UpperCAmelCase_=64 ,			UpperCAmelCase_=0.0 ,			UpperCAmelCase_=1 ,			UpperCAmelCase_=0 ,			UpperCAmelCase_=4096 ,			UpperCAmelCase_=12 ,			UpperCAmelCase_=None ,			UpperCAmelCase_=1 ,			UpperCAmelCase_=0 ,			UpperCAmelCase_=320 ,			UpperCAmelCase_=False ,			UpperCAmelCase_=True ,			UpperCAmelCase_=None ,			**UpperCAmelCase_ ,			):
								lowerCamelCase									=hash_seed
								lowerCamelCase									=vocab_size
								lowerCamelCase									=attention_head_size
								lowerCamelCase									=hidden_size
								lowerCamelCase									=num_attention_heads
								lowerCamelCase									=num_hashes
								lowerCamelCase									=len(UpperCAmelCase_				)
								lowerCamelCase									=tuple(UpperCAmelCase_				) if isinstance(UpperCAmelCase_ ,			UpperCAmelCase_				) else num_buckets
								lowerCamelCase									=lsh_attn_chunk_length
								lowerCamelCase									=local_attn_chunk_length
								lowerCamelCase									=lsh_num_chunks_after
								lowerCamelCase									=lsh_num_chunks_before
								lowerCamelCase									=local_num_chunks_after
								lowerCamelCase									=local_num_chunks_before
								lowerCamelCase									=hidden_act
								lowerCamelCase									=feed_forward_size
								lowerCamelCase									=hidden_dropout_prob
								lowerCamelCase									=lsh_attention_probs_dropout_prob
								lowerCamelCase									=local_attention_probs_dropout_prob
								lowerCamelCase									=max_position_embeddings
								lowerCamelCase									=initializer_range
								lowerCamelCase									=layer_norm_eps
								lowerCamelCase									=axial_pos_embds
								lowerCamelCase									=tuple(UpperCAmelCase_				)
								lowerCamelCase									=tuple(UpperCAmelCase_				)
								lowerCamelCase									=axial_norm_std
								lowerCamelCase									=chunk_size_lm_head
								lowerCamelCase									=attn_layers
								lowerCamelCase									=use_cache
								lowerCamelCase									=classifier_dropout
								super().__init__(
								    pad_token_id=UpperCAmelCase_ ,			eos_token_id=UpperCAmelCase_ ,			is_decoder=UpperCAmelCase_ ,			tie_word_embeddings=UpperCAmelCase_ ,			**UpperCAmelCase_ ,			)












import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


UpperCAmelCase__    :      int       					=logging.get_logger(__name__)

UpperCAmelCase__    :      Optional[Any]       					={'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCAmelCase__    :      List[str]       					={
    '''vocab_file''': {
        '''microsoft/layoutlm-base-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt'''
        ),
        '''microsoft/layoutlm-large-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''microsoft/layoutlm-base-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json'''
        ),
        '''microsoft/layoutlm-large-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCAmelCase__    :      Dict       					={
    '''microsoft/layoutlm-base-uncased''': 5_12,
    '''microsoft/layoutlm-large-uncased''': 5_12,
}

UpperCAmelCase__    :      List[Any]       					={
    '''microsoft/layoutlm-base-uncased''': {'''do_lower_case''': True},
    '''microsoft/layoutlm-large-uncased''': {'''do_lower_case''': True},
}

class    __A  (    a     ):
				__A  =					VOCAB_FILES_NAMES
				__A  =					PRETRAINED_VOCAB_FILES_MAP
				__A  =					PRETRAINED_INIT_CONFIGURATION
				__A  =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__A  =					LayoutLMTokenizer


				def __init__(		self ,			UpperCAmelCase_=None ,			UpperCAmelCase_=None ,			UpperCAmelCase_=True ,			UpperCAmelCase_="[UNK]" ,			UpperCAmelCase_="[SEP]" ,			UpperCAmelCase_="[PAD]" ,			UpperCAmelCase_="[CLS]" ,			UpperCAmelCase_="[MASK]" ,			UpperCAmelCase_=True ,			UpperCAmelCase_=None ,			**UpperCAmelCase_ ,			):
								super().__init__(
								    UpperCAmelCase_ ,			tokenizer_file=UpperCAmelCase_ ,			do_lower_case=UpperCAmelCase_ ,			unk_token=UpperCAmelCase_ ,			sep_token=UpperCAmelCase_ ,			pad_token=UpperCAmelCase_ ,			cls_token=UpperCAmelCase_ ,			mask_token=UpperCAmelCase_ ,			tokenize_chinese_chars=UpperCAmelCase_ ,			strip_accents=UpperCAmelCase_ ,			**UpperCAmelCase_ ,			)

								lowerCamelCase									=json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
								if (
								    normalizer_state.get("""lowercase""" ,			UpperCAmelCase_				) != do_lower_case
								    or normalizer_state.get("""strip_accents""" ,			UpperCAmelCase_				) != strip_accents
								    or normalizer_state.get("""handle_chinese_chars""" ,			UpperCAmelCase_				) != tokenize_chinese_chars
								):
												lowerCamelCase									=getattr(UpperCAmelCase_ ,			normalizer_state.pop("""type"""				)				)
												lowerCamelCase									=do_lower_case
												lowerCamelCase									=strip_accents
												lowerCamelCase									=tokenize_chinese_chars
												lowerCamelCase									=normalizer_class(**UpperCAmelCase_				)

								lowerCamelCase									=do_lower_case


				def _snake_case							(		self ,			UpperCAmelCase_ ,			UpperCAmelCase_=None				):
								lowerCamelCase									=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
												output += token_ids_a + [self.sep_token_id]

								return output


				def _snake_case							(		self ,			UpperCAmelCase_ ,			UpperCAmelCase_ = None				):
								lowerCamelCase									=[self.sep_token_id]
								lowerCamelCase									=[self.cls_token_id]
								if token_ids_a is None:
												return len(cls + token_ids_a + sep				) * [0]
								return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]


				def _snake_case							(		self ,			UpperCAmelCase_ ,			UpperCAmelCase_ = None				):
								lowerCamelCase									=self._tokenizer.model.save(UpperCAmelCase_ ,			name=UpperCAmelCase_				)
								return tuple(UpperCAmelCase_				)





