
'''simple docstring'''





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


UpperCAmelCase            = logging.get_logger(__name__)

UpperCAmelCase            = {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase            = {
    "vocab_file": {
        "microsoft/layoutlm-base-uncased": (
            "https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt"
        ),
        "microsoft/layoutlm-large-uncased": (
            "https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "microsoft/layoutlm-base-uncased": (
            "https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json"
        ),
        "microsoft/layoutlm-large-uncased": (
            "https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json"
        ),
    },
}

UpperCAmelCase            = {
    "microsoft/layoutlm-base-uncased": 512,
    "microsoft/layoutlm-large-uncased": 512,
}

UpperCAmelCase            = {
    "microsoft/layoutlm-base-uncased": {"do_lower_case": True},
    "microsoft/layoutlm-large-uncased": {"do_lower_case": True},
}

class     a (					__magic_name__     ):
				_snake_case		     =			VOCAB_FILES_NAMES
				_snake_case		     =			PRETRAINED_VOCAB_FILES_MAP
				_snake_case		     =			PRETRAINED_INIT_CONFIGURATION
				_snake_case		     =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				_snake_case		     =			LayoutLMTokenizer




				def __init__(				self						:					Optional[Any],     SCREAMING_SNAKE_CASE_						:					List[Any]=None,     SCREAMING_SNAKE_CASE_						:					Dict=None,     SCREAMING_SNAKE_CASE_						:					Tuple=True,     SCREAMING_SNAKE_CASE_						:					List[Any]="[UNK]",     SCREAMING_SNAKE_CASE_						:					Dict="[SEP]",     SCREAMING_SNAKE_CASE_						:					Dict="[PAD]",     SCREAMING_SNAKE_CASE_						:					int="[CLS]",     SCREAMING_SNAKE_CASE_						:					Union[str, Any]="[MASK]",     SCREAMING_SNAKE_CASE_						:					List[Any]=True,     SCREAMING_SNAKE_CASE_						:					Optional[int]=None,     **SCREAMING_SNAKE_CASE_						:					str,     ):
										super().__init__(
										    SCREAMING_SNAKE_CASE_,     tokenizer_file=SCREAMING_SNAKE_CASE_,     do_lower_case=SCREAMING_SNAKE_CASE_,     unk_token=SCREAMING_SNAKE_CASE_,     sep_token=SCREAMING_SNAKE_CASE_,     pad_token=SCREAMING_SNAKE_CASE_,     cls_token=SCREAMING_SNAKE_CASE_,     mask_token=SCREAMING_SNAKE_CASE_,     tokenize_chinese_chars=SCREAMING_SNAKE_CASE_,     strip_accents=SCREAMING_SNAKE_CASE_,     **SCREAMING_SNAKE_CASE_,     )

										snake_case							:  Optional[Any]			=     json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
										if (
										    normalizer_state.get('''lowercase''',     SCREAMING_SNAKE_CASE_   ) != do_lower_case
										    or normalizer_state.get('''strip_accents''',     SCREAMING_SNAKE_CASE_   ) != strip_accents
										    or normalizer_state.get('''handle_chinese_chars''',     SCREAMING_SNAKE_CASE_   ) != tokenize_chinese_chars
										):
																snake_case							:  List[str]			=     getattr(SCREAMING_SNAKE_CASE_,     normalizer_state.pop('''type'''   )   )
																snake_case							:  Dict			=     do_lower_case
																snake_case							:  List[str]			=     strip_accents
																snake_case							:  List[Any]			=     tokenize_chinese_chars
																snake_case							:  Optional[int]			=     normalizer_class(**SCREAMING_SNAKE_CASE_   )

										snake_case							:  List[str]			=     do_lower_case




				def 						__snake_case				(				self						:					str,     SCREAMING_SNAKE_CASE_						:					int,     SCREAMING_SNAKE_CASE_						:					Optional[int]=None   ):
										snake_case							:  str			=     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

										if token_ids_a:
																output += token_ids_a + [self.sep_token_id]

										return output




				def 						__snake_case				(				self						:					str,     SCREAMING_SNAKE_CASE_						:					List[int],     SCREAMING_SNAKE_CASE_						:					Optional[List[int]] = None   ):
										snake_case							:  str			=     [self.sep_token_id]
										snake_case							:  Tuple			=     [self.cls_token_id]
										if token_ids_a is None:
																return len(cls + token_ids_a + sep   ) * [0]
										return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]




				def 						__snake_case				(				self						:					str,     SCREAMING_SNAKE_CASE_						:					str,     SCREAMING_SNAKE_CASE_						:					Optional[str] = None   ):
										snake_case							:  str			=     self._tokenizer.model.save(SCREAMING_SNAKE_CASE_,     name=SCREAMING_SNAKE_CASE_   )
										return tuple(SCREAMING_SNAKE_CASE_   )







"""simple docstring"""



import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
      import torch

if is_vision_available():
      from PIL import Image

      from transformers import ViTImageProcessor




class       UpperCAmelCase_    (		unittest.TestCase				):




    def __init__(       self		:			Union[str, Any]      ,					snake_case_		:			Tuple      ,					snake_case_		:			Dict=7      ,					snake_case_		:			List[Any]=3      ,					snake_case_		:			Optional[int]=18      ,					snake_case_		:			Tuple=30      ,					snake_case_		:			Dict=400      ,					snake_case_		:			List[str]=True      ,					snake_case_		:			Optional[int]=None      ,					snake_case_		:			List[str]=True      ,					snake_case_		:			Optional[Any]=[0.5, 0.5, 0.5]      ,					snake_case_		:			Optional[int]=[0.5, 0.5, 0.5]      ,					) ->				List[Any]:




     '''simple docstring'''

     A__			=							size if size is not None else {"height": 18, "width": 18}
     A__			=							parent
     A__			=							batch_size
     A__			=							num_channels
     A__			=							image_size
     A__			=							min_resolution
     A__			=							max_resolution
     A__			=							do_resize
     A__			=							size
     A__			=							do_normalize
     A__			=							image_mean
     A__			=							image_std




    def 				__magic_name__				(       self		:			Union[str, Any] ) ->				Dict:




     '''simple docstring'''

     return {
         "image_mean": self.image_mean,
         "image_std": self.image_std,
         "do_normalize": self.do_normalize,
         "do_resize": self.do_resize,
         "size": self.size,
     }






@require_torch
@require_vision
class       UpperCAmelCase_    (		A_,  unittest.TestCase				):
    lowercase__		   = ViTImageProcessor if is_vision_available() else None




    def 				__magic_name__				(       self		:			int ) ->				Any:




     '''simple docstring'''

     A__			=							ViTImageProcessingTester(self )




    @property
    def 				__magic_name__				(       self		:			Optional[Any] ) ->				str:




     '''simple docstring'''

     return self.image_processor_tester.prepare_image_processor_dict()




    def 				__magic_name__				(       self		:			Tuple ) ->				str:




     '''simple docstring'''

     A__			=							self.image_processing_class(**self.image_processor_dict )
     self.assertTrue(hasattr(snake_case_      ,					"image_mean" ) )
     self.assertTrue(hasattr(snake_case_      ,					"image_std" ) )
     self.assertTrue(hasattr(snake_case_      ,					"do_normalize" ) )
     self.assertTrue(hasattr(snake_case_      ,					"do_resize" ) )
     self.assertTrue(hasattr(snake_case_      ,					"size" ) )




    def 				__magic_name__				(       self		:			Union[str, Any] ) ->				List[str]:




     '''simple docstring'''

     A__			=							self.image_processing_class.from_dict(self.image_processor_dict )
     self.assertEqual(image_processor.size      ,					{"height": 18, "width": 18} )

     A__			=							self.image_processing_class.from_dict(self.image_processor_dict      ,					size=42 )
     self.assertEqual(image_processor.size      ,					{"height": 42, "width": 42} )




    def 				__magic_name__				(       self		:			Any ) ->				Optional[Any]:




     '''simple docstring'''

     pass




    def 				__magic_name__				(       self		:			Optional[int] ) ->				Any:




     '''simple docstring'''

     A__			=							self.image_processing_class(**self.image_processor_dict )
     # create random PIL images
     A__			=							prepare_image_inputs(self.image_processor_tester      ,					equal_resolution=snake_case_ )
     for image in image_inputs:
      self.assertIsInstance(snake_case_      ,					Image.Image )

     # Test not batched input
     A__			=							image_processing(image_inputs[0]      ,					return_tensors="pt" ).pixel_values
     self.assertEqual(
         encoded_images.shape      ,					(
             1,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.size["height"],
             self.image_processor_tester.size["width"],
         )      ,					)

     # Test batched
     A__			=							image_processing(snake_case_      ,					return_tensors="pt" ).pixel_values
     self.assertEqual(
         encoded_images.shape      ,					(
             self.image_processor_tester.batch_size,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.size["height"],
             self.image_processor_tester.size["width"],
         )      ,					)




    def 				__magic_name__				(       self		:			str ) ->				Optional[Any]:




     '''simple docstring'''

     A__			=							self.image_processing_class(**self.image_processor_dict )
     # create random numpy tensors
     A__			=							prepare_image_inputs(self.image_processor_tester      ,					equal_resolution=snake_case_      ,					numpify=snake_case_ )
     for image in image_inputs:
      self.assertIsInstance(snake_case_      ,					np.ndarray )

     # Test not batched input
     A__			=							image_processing(image_inputs[0]      ,					return_tensors="pt" ).pixel_values
     self.assertEqual(
         encoded_images.shape      ,					(
             1,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.size["height"],
             self.image_processor_tester.size["width"],
         )      ,					)

     # Test batched
     A__			=							image_processing(snake_case_      ,					return_tensors="pt" ).pixel_values
     self.assertEqual(
         encoded_images.shape      ,					(
             self.image_processor_tester.batch_size,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.size["height"],
             self.image_processor_tester.size["width"],
         )      ,					)




    def 				__magic_name__				(       self		:			Dict ) ->				Union[str, Any]:




     '''simple docstring'''

     A__			=							self.image_processing_class(**self.image_processor_dict )
     # create random PyTorch tensors
     A__			=							prepare_image_inputs(self.image_processor_tester      ,					equal_resolution=snake_case_      ,					torchify=snake_case_ )
     for image in image_inputs:
      self.assertIsInstance(snake_case_      ,					torch.Tensor )

     # Test not batched input
     A__			=							image_processing(image_inputs[0]      ,					return_tensors="pt" ).pixel_values
     self.assertEqual(
         encoded_images.shape      ,					(
             1,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.size["height"],
             self.image_processor_tester.size["width"],
         )      ,					)

     # Test batched
     A__			=							image_processing(snake_case_      ,					return_tensors="pt" ).pixel_values
     self.assertEqual(
         encoded_images.shape      ,					(
             self.image_processor_tester.batch_size,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.size["height"],
             self.image_processor_tester.size["width"],
         )      ,					)





