





from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PretrainedConfig, PreTrainedTokenizer
from ...onnx import OnnxConfig, PatchingSpec
from ...utils import TensorType, is_torch_available, logging


SCREAMING_SNAKE_CASE_		  =			logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_		  =			{
    """microsoft/layoutlm-base-uncased""": (
        """https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/config.json"""
    ),
    """microsoft/layoutlm-large-uncased""": (
        """https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/config.json"""
    ),
}

class 						UpperCamelCase__					( lowerCAmelCase_  ):



     '''simple docstring'''

     __snake_case       :      int						      =     "layoutlm"






     def __init__(				self		:       int		,lowerCamelCase__		:       Optional[Any]=30522		,lowerCamelCase__		:       Optional[Any]=768		,lowerCamelCase__		:       Dict=12		,lowerCamelCase__		:       Tuple=12		,lowerCamelCase__		:       Union[str, Any]=3072		,lowerCamelCase__		:       Dict="gelu"		,lowerCamelCase__		:       Optional[int]=0.1		,lowerCamelCase__		:       Tuple=0.1		,lowerCamelCase__		:       int=512		,lowerCamelCase__		:       int=2		,lowerCamelCase__		:       Any=0.02		,lowerCamelCase__		:       Union[str, Any]=1e-1_2		,lowerCamelCase__		:       List[Any]=0		,lowerCamelCase__		:       Tuple="absolute"		,lowerCamelCase__		:       str=True		,lowerCamelCase__		:       Optional[Any]=1024		,**lowerCamelCase__		:       Union[str, Any]		,)	->						Any:




           '''simple docstring'''


           super().__init__(pad_token_id=lowerCamelCase__		,**lowerCamelCase__			)
           SCREAMING_SNAKE_CASE						=		vocab_size
           SCREAMING_SNAKE_CASE						=		hidden_size
           SCREAMING_SNAKE_CASE						=		num_hidden_layers
           SCREAMING_SNAKE_CASE						=		num_attention_heads
           SCREAMING_SNAKE_CASE						=		hidden_act
           SCREAMING_SNAKE_CASE						=		intermediate_size
           SCREAMING_SNAKE_CASE						=		hidden_dropout_prob
           SCREAMING_SNAKE_CASE						=		attention_probs_dropout_prob
           SCREAMING_SNAKE_CASE						=		max_position_embeddings
           SCREAMING_SNAKE_CASE						=		type_vocab_size
           SCREAMING_SNAKE_CASE						=		initializer_range
           SCREAMING_SNAKE_CASE						=		layer_norm_eps
           SCREAMING_SNAKE_CASE						=		position_embedding_type
           SCREAMING_SNAKE_CASE						=		use_cache
           SCREAMING_SNAKE_CASE						=		max_ad_position_embeddings

class 						UpperCamelCase__					( lowerCAmelCase_  ):



     '''simple docstring'''

     def __init__(				self		:       int		,lowerCamelCase__		:       PretrainedConfig		,lowerCamelCase__		:       str = "default"		,lowerCamelCase__		:       List[PatchingSpec] = None		,)	->						Optional[int]:




           '''simple docstring'''


           super().__init__(lowerCamelCase__		,task=lowerCamelCase__		,patching_specs=lowerCamelCase__			)
           SCREAMING_SNAKE_CASE						=		config.max_ad_position_embeddings - 1






     @property
     def 							SCREAMING_SNAKE_CASE__    (				self		:       Dict			)	->						Mapping[str, Mapping[int, str]]:




           '''simple docstring'''


           return OrderedDict(
               [
                   ("""input_ids""", {0: """batch""", 1: """sequence"""}),
                   ("""bbox""", {0: """batch""", 1: """sequence"""}),
                   ("""attention_mask""", {0: """batch""", 1: """sequence"""}),
                   ("""token_type_ids""", {0: """batch""", 1: """sequence"""}),
               ]			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[str]		,lowerCamelCase__		:       PreTrainedTokenizer		,lowerCamelCase__		:       int = -1		,lowerCamelCase__		:       int = -1		,lowerCamelCase__		:       bool = False		,lowerCamelCase__		:       Optional[TensorType] = None		,)	->						Mapping[str, Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		super().generate_dummy_inputs(
               lowerCamelCase__		,batch_size=lowerCamelCase__		,seq_length=lowerCamelCase__		,is_pair=lowerCamelCase__		,framework=lowerCamelCase__			)

           # Generate a dummy bbox
           SCREAMING_SNAKE_CASE						=		[48, 84, 73, 128]

           if not framework == TensorType.PYTORCH:
                 raise NotImplementedError("""Exporting LayoutLM to ONNX is currently only supported for PyTorch."""			)

           if not is_torch_available():
                 raise ValueError("""Cannot generate dummy inputs without PyTorch installed."""			)
           import torch

           SCREAMING_SNAKE_CASE,   SCREAMING_SNAKE_CASE						=		input_dict["""input_ids"""].shape
           SCREAMING_SNAKE_CASE						=		torch.tensor([*[box] * seq_length]			).tile(lowerCamelCase__		,1		,1			)
           return input_dict









import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 						UpperCamelCase__					( lowerCAmelCase_				, unittest.TestCase  ):



     '''simple docstring'''

     __snake_case       :      Dict						      =     PreTrainedTokenizerFast
     __snake_case       :      Optional[int]						      =     False
     __snake_case       :      List[Any]						      =     True
     __snake_case       :      str						      =     "tokenizer_file"






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Union[str, Any]			)	->						Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		False  # because we don't have pretrained_vocab_files_map
           super().setUp()
           SCREAMING_SNAKE_CASE						=		True

           SCREAMING_SNAKE_CASE						=		["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
           SCREAMING_SNAKE_CASE						=		"""SaulLu/dummy-tokenizer-bytelevel-bpe"""

           # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
           SCREAMING_SNAKE_CASE						=		[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

           SCREAMING_SNAKE_CASE						=		PreTrainedTokenizerFast.from_pretrained(model_paths[0]			)
           tokenizer.save_pretrained(self.tmpdirname			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[Any]			)	->						Tuple:




           '''simple docstring'''


           pass






     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[str]			)	->						List[Any]:




           '''simple docstring'''


           pass






     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[Any]			)	->						Union[str, Any]:




           '''simple docstring'''


           pass






     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[str]			)	->						List[Any]:




           '''simple docstring'''


           pass






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Any			)	->						Optional[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		self.tmpdirname
           # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
                       try:
                             SCREAMING_SNAKE_CASE						=		tempfile.mkdtemp()
                             SCREAMING_SNAKE_CASE						=		self.rust_tokenizer_class.from_pretrained(lowerCamelCase__		,**lowerCamelCase__			)

                             tokenizer.save_pretrained(self.tmpdirname			)
                             super().test_training_new_tokenizer()
                       finally:
                             # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                             # is restored
                             shutil.rmtree(self.tmpdirname			)
                             SCREAMING_SNAKE_CASE						=		tmpdirname_orig






     def 							SCREAMING_SNAKE_CASE__    (				self		:       int			)	->						Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		self.tmpdirname
           # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
                       try:
                             SCREAMING_SNAKE_CASE						=		tempfile.mkdtemp()
                             SCREAMING_SNAKE_CASE						=		self.rust_tokenizer_class.from_pretrained(lowerCamelCase__		,**lowerCamelCase__			)

                             tokenizer.save_pretrained(self.tmpdirname			)
                             super().test_training_new_tokenizer_with_special_tokens_change()
                       finally:
                             # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                             # is restored
                             shutil.rmtree(self.tmpdirname			)
                             SCREAMING_SNAKE_CASE						=		tmpdirname_orig






     def 							SCREAMING_SNAKE_CASE__    (				self		:       int			)	->						Tuple:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name			)

           SCREAMING_SNAKE_CASE						=		("""a""" for _ in range(1000			))
           SCREAMING_SNAKE_CASE						=		tokenizer.train_new_from_iterator(text_iterator=lowerCamelCase__		,length=1000		,vocab_size=50			)

           SCREAMING_SNAKE_CASE						=		new_tokenizer.encode("""a🤗"""			)
           self.assertEqual(lowerCamelCase__		,[64, 172, 253, 97, 245]			)

@require_tokenizers
class 						UpperCamelCase__					( unittest.TestCase  ):



     '''simple docstring'''

     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[Any]			)	->						str:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		AutoTokenizer.from_pretrained("""bert-base-cased"""			)
           SCREAMING_SNAKE_CASE						=		json.loads(tokenizer._tokenizer.to_str()			)
           SCREAMING_SNAKE_CASE						=		len(lowerCamelCase__			)

           with tempfile.TemporaryDirectory() as tmp_dir:
                 # Hack to save this in the tokenizer_config.json
                 SCREAMING_SNAKE_CASE						=		["""tokenizer.4.0.0.json"""]
                 tokenizer.save_pretrained(lowerCamelCase__			)
                 json.dump(lowerCamelCase__		,open(os.path.join(lowerCamelCase__		,"""tokenizer.4.0.0.json"""			)		,"""w"""			)			)

                 # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
                 SCREAMING_SNAKE_CASE						=		AutoTokenizer.from_pretrained(lowerCamelCase__			)
                 self.assertEqual(len(lowerCamelCase__			)		,len(lowerCamelCase__			) + 1			)
                 SCREAMING_SNAKE_CASE						=		json.loads(new_tokenizer._tokenizer.to_str()			)
                 self.assertIn("""huggingface"""		,json_tokenizer["""model"""]["""vocab"""]			)

                 # Will need to be adjusted if we reach v42 and this test is still here.
                 # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
                 shutil.move(os.path.join(lowerCamelCase__		,"""tokenizer.4.0.0.json"""			)		,os.path.join(lowerCamelCase__		,"""tokenizer.42.0.0.json"""			)			)
                 SCREAMING_SNAKE_CASE						=		["""tokenizer.42.0.0.json"""]
                 tokenizer.save_pretrained(lowerCamelCase__			)
                 SCREAMING_SNAKE_CASE						=		AutoTokenizer.from_pretrained(lowerCamelCase__			)
                 self.assertEqual(len(lowerCamelCase__			)		,len(lowerCamelCase__			)			)
                 SCREAMING_SNAKE_CASE						=		json.loads(new_tokenizer._tokenizer.to_str()			)
                 self.assertNotIn("""huggingface"""		,json_tokenizer["""model"""]["""vocab"""]			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Dict			)	->						Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		"""hf-internal-testing/test-two-tokenizers"""

           # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
           SCREAMING_SNAKE_CASE						=		AutoTokenizer.from_pretrained(lowerCamelCase__			)
           self.assertEqual(len(lowerCamelCase__			)		,28997			)
           SCREAMING_SNAKE_CASE						=		json.loads(tokenizer._tokenizer.to_str()			)
           self.assertIn("""huggingface"""		,json_tokenizer["""model"""]["""vocab"""]			)

           # Testing an older version by monkey-patching the version in the module it's used.
           import transformers as old_transformers

           SCREAMING_SNAKE_CASE						=		"""3.0.0"""
           SCREAMING_SNAKE_CASE						=		old_transformers.models.auto.AutoTokenizer.from_pretrained(lowerCamelCase__			)
           self.assertEqual(len(lowerCamelCase__			)		,28996			)
           SCREAMING_SNAKE_CASE						=		json.loads(old_tokenizer._tokenizer.to_str()			)
           self.assertNotIn("""huggingface"""		,json_tokenizer["""model"""]["""vocab"""]			)

@require_tokenizers
class 						UpperCamelCase__					( unittest.TestCase  ):



     '''simple docstring'''

     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[str]			)	->						Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""			)
           SCREAMING_SNAKE_CASE						=		"""The Matrix is a 1999 science fiction action film."""

           with concurrent.futures.ThreadPoolExecutor() as executor:
                 SCREAMING_SNAKE_CASE						=		[executor.submit(self.fetch		,lowerCamelCase__		,lowerCamelCase__			) for i in range(10			)]
                 SCREAMING_SNAKE_CASE						=		[future.result() for future in futures]
                 self.assertEqual(lowerCamelCase__		,[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10			)]			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Optional[int]		,lowerCamelCase__		:       Optional[int]		,lowerCamelCase__		:       str			)	->						Union[str, Any]:




           '''simple docstring'''


           return tokenizer.encode(lowerCamelCase__		,truncation="""longest_first"""		,padding="""longest"""			)


