
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


__UpperCAmelCase   						=     logging.get_logger(__name__)

__UpperCAmelCase   						=     {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__UpperCAmelCase   						=     {
    "vocab_file": {
        "microsoft/layoutlm-base-uncased": (
            "https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt"
        ),
        "microsoft/layoutlm-large-uncased": (
            "https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "microsoft/layoutlm-base-uncased": (
            "https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json"
        ),
        "microsoft/layoutlm-large-uncased": (
            "https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json"
        ),
    },
}

__UpperCAmelCase   						=     {
    "microsoft/layoutlm-base-uncased": 5_12,
    "microsoft/layoutlm-large-uncased": 5_12,
}

__UpperCAmelCase   						=     {
    "microsoft/layoutlm-base-uncased": {"do_lower_case": True},
    "microsoft/layoutlm-large-uncased": {"do_lower_case": True},
}





class 				UpperCamelCase__		(	__SCREAMING_SNAKE_CASE	):






       """simple docstring"""


       UpperCAmelCase_				       =VOCAB_FILES_NAMES
       UpperCAmelCase_				       =PRETRAINED_VOCAB_FILES_MAP
       UpperCAmelCase_				       =PRETRAINED_INIT_CONFIGURATION
       UpperCAmelCase_				       =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       UpperCAmelCase_				       =LayoutLMTokenizer



       def __init__(							self ,     _A=None ,     _A=None ,     _A=True ,     _A="[UNK]" ,     _A="[SEP]" ,     _A="[PAD]" ,     _A="[CLS]" ,     _A="[MASK]" ,     _A=True ,     _A=None ,     **_A ,     )				->				List[Any]:
          super().__init__(
              _A ,     tokenizer_file=_A ,     do_lower_case=_A ,     unk_token=_A ,     sep_token=_A ,     pad_token=_A ,     cls_token=_A ,     mask_token=_A ,     tokenize_chinese_chars=_A ,     strip_accents=_A ,     **_A ,     )

          SCREAMING_SNAKE_CASE_	=      json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
          if (
              normalizer_state.get('''lowercase''' ,     _A	) != do_lower_case
              or normalizer_state.get('''strip_accents''' ,     _A	) != strip_accents
              or normalizer_state.get('''handle_chinese_chars''' ,     _A	) != tokenize_chinese_chars
          ):
             SCREAMING_SNAKE_CASE_	=      getattr(_A ,     normalizer_state.pop('''type'''	)	)
             SCREAMING_SNAKE_CASE_	=      do_lower_case
             SCREAMING_SNAKE_CASE_	=      strip_accents
             SCREAMING_SNAKE_CASE_	=      tokenize_chinese_chars
             SCREAMING_SNAKE_CASE_	=      normalizer_class(**_A	)

          SCREAMING_SNAKE_CASE_	=      do_lower_case



       def     _UpperCamelCase	(							self ,     _A ,     _A=None	)				->				Optional[int]:
          SCREAMING_SNAKE_CASE_	=      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
             output += token_ids_a + [self.sep_token_id]

          return output



       def     _UpperCamelCase	(							self ,     _A ,     _A = None	)				->				List[int]:
          SCREAMING_SNAKE_CASE_	=      [self.sep_token_id]
          SCREAMING_SNAKE_CASE_	=      [self.cls_token_id]
          if token_ids_a is None:
             return len(cls + token_ids_a + sep	) * [0]
          return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]




       def     _UpperCamelCase	(							self ,     _A ,     _A = None	)				->				Tuple[str]:
          SCREAMING_SNAKE_CASE_	=      self._tokenizer.model.save(_A ,     name=_A	)
          return tuple(_A	)




import torch

from transformers import PreTrainedModel

from .custom_configuration import CustomConfig, NoSuperInitConfig





class 				UpperCamelCase__		(	__SCREAMING_SNAKE_CASE	):






       """simple docstring"""


       UpperCAmelCase_				       =CustomConfig



       def __init__(							self ,     _A	)				->				int:
          super().__init__(_A	)
          SCREAMING_SNAKE_CASE_	=      torch.nn.Linear(config.hidden_size ,     config.hidden_size	)



       def     _UpperCamelCase	(							self ,     _A	)				->				Optional[int]:
          return self.linear(_A	)




       def     _UpperCamelCase	(							self ,     _A	)				->				int:
          pass







class 				UpperCamelCase__		(	__SCREAMING_SNAKE_CASE	):






       """simple docstring"""


       UpperCAmelCase_				       =NoSuperInitConfig



       def __init__(							self ,     _A	)				->				Optional[int]:
          super().__init__(_A	)
          SCREAMING_SNAKE_CASE_	=      torch.nn.Linear(config.attribute ,     config.attribute	)



       def     _UpperCamelCase	(							self ,     _A	)				->				Any:
          return self.linear(_A	)




       def     _UpperCamelCase	(							self ,     _A	)				->				Tuple:
          pass


