






import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer


class 		lowerCAmelCase_				(  tf.keras.layers.Layer ):





     def __init__(	self							, _UpperCAmelCase							, _UpperCAmelCase							, _UpperCAmelCase = None							, _UpperCAmelCase = None							, _UpperCAmelCase = None							, _UpperCAmelCase = "longest"							, _UpperCAmelCase = True							, _UpperCAmelCase = 512							, _UpperCAmelCase = None							, _UpperCAmelCase = True							, _UpperCAmelCase = True							, _UpperCAmelCase = True							, **_UpperCAmelCase							, )		->				Union[str, Any]:
        super().__init__()
        if use_fast_bert_tokenizer:
           lowerCamelCase									=FastBertTokenizer(
               _UpperCAmelCase							, token_out_type=tf.intaa							, lower_case_nfd_strip_accents=_UpperCAmelCase							, **_UpperCAmelCase  )
        else:
           lowerCamelCase									=tf.lookup.StaticVocabularyTable(
               tf.lookup.KeyValueTensorInitializer(
                   keys=_UpperCAmelCase							, key_dtype=tf.string							, values=tf.range(tf.size(_UpperCAmelCase							, out_type=tf.intaa  )							, dtype=tf.intaa  )							, value_dtype=tf.intaa							, )							, num_oov_buckets=1							, )
           lowerCamelCase									=BertTokenizerLayer(
               _UpperCAmelCase							, token_out_type=tf.intaa							, lower_case=_UpperCAmelCase							, **_UpperCAmelCase  )

        lowerCamelCase									=vocab_list
        lowerCamelCase									=do_lower_case
        lowerCamelCase									=cls_token_id or vocab_list.index("""[CLS]"""  )
        lowerCamelCase									=sep_token_id or vocab_list.index("""[SEP]"""  )
        lowerCamelCase									=pad_token_id or vocab_list.index("""[PAD]"""  )
        lowerCamelCase									=ShrinkLongestTrimmer(max_length - 3							, axis=1  )  # Allow room for special tokens
        lowerCamelCase									=max_length
        lowerCamelCase									=padding
        lowerCamelCase									=truncation
        lowerCamelCase									=pad_to_multiple_of
        lowerCamelCase									=return_token_type_ids
        lowerCamelCase									=return_attention_mask





     @classmethod
     def 						lowerCamelCase							(	cls							, _UpperCAmelCase							, **_UpperCAmelCase  )		->				Optional[Any]:  # noqa: F821
        lowerCamelCase									=kwargs.pop("""do_lower_case"""							, _UpperCAmelCase  )
        lowerCamelCase									=tokenizer.do_lower_case if do_lower_case is None else do_lower_case
        lowerCamelCase									=kwargs.pop("""cls_token_id"""							, _UpperCAmelCase  )
        lowerCamelCase									=tokenizer.cls_token_id if cls_token_id is None else cls_token_id
        lowerCamelCase									=kwargs.pop("""sep_token_id"""							, _UpperCAmelCase  )
        lowerCamelCase									=tokenizer.sep_token_id if sep_token_id is None else sep_token_id
        lowerCamelCase									=kwargs.pop("""pad_token_id"""							, _UpperCAmelCase  )
        lowerCamelCase									=tokenizer.pad_token_id if pad_token_id is None else pad_token_id

        lowerCamelCase									=tokenizer.get_vocab()
        lowerCamelCase									=sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]							, key=lambda _UpperCAmelCase  : x[1]  )
        lowerCamelCase									=[entry[0] for entry in vocab]
        return cls(
            vocab_list=_UpperCAmelCase							, do_lower_case=_UpperCAmelCase							, cls_token_id=_UpperCAmelCase							, sep_token_id=_UpperCAmelCase							, pad_token_id=_UpperCAmelCase							, **_UpperCAmelCase							, )





     @classmethod
     def 						lowerCamelCase							(	cls							, _UpperCAmelCase							, *_UpperCAmelCase							, **_UpperCAmelCase  )		->				Optional[Any]:
        try:
           lowerCamelCase									=BertTokenizer.from_pretrained(_UpperCAmelCase							, *_UpperCAmelCase							, **_UpperCAmelCase  )
        except:  # noqa: E722
           from .tokenization_bert_fast import BertTokenizerFast

           lowerCamelCase									=BertTokenizerFast.from_pretrained(_UpperCAmelCase							, *_UpperCAmelCase							, **_UpperCAmelCase  )
        return cls.from_tokenizer(_UpperCAmelCase							, **_UpperCAmelCase  )





     def 						lowerCamelCase							(	self							, _UpperCAmelCase  )		->				Any:
        if self.do_lower_case:
           lowerCamelCase									=case_fold_utfa(_UpperCAmelCase  )
        lowerCamelCase									=self.tf_tokenizer.tokenize(_UpperCAmelCase  )
        return tokens.merge_dims(1							, -1  )





     def 						lowerCamelCase							(	self							, _UpperCAmelCase							, _UpperCAmelCase=None							, _UpperCAmelCase=None							, _UpperCAmelCase=None							, _UpperCAmelCase=None							, _UpperCAmelCase=None							, _UpperCAmelCase=None							, _UpperCAmelCase=None							, )		->				int:
        if padding is None:
           lowerCamelCase									=self.padding
        if padding not in ("longest", "max_length"):
           raise ValueError("""Padding must be either 'longest' or 'max_length'!"""  )
        if max_length is not None and text_pair is not None:
           # Because we have to instantiate a Trimmer to do it properly
           raise ValueError("""max_length cannot be overridden at call time when truncating paired texts!"""  )
        if max_length is None:
           lowerCamelCase									=self.max_length
        if truncation is None:
           lowerCamelCase									=self.truncation
        if pad_to_multiple_of is None:
           lowerCamelCase									=self.pad_to_multiple_of
        if return_token_type_ids is None:
           lowerCamelCase									=self.return_token_type_ids
        if return_attention_mask is None:
           lowerCamelCase									=self.return_attention_mask
        if not isinstance(_UpperCAmelCase							, tf.Tensor  ):
           lowerCamelCase									=tf.convert_to_tensor(_UpperCAmelCase  )
        if text_pair is not None and not isinstance(_UpperCAmelCase							, tf.Tensor  ):
           lowerCamelCase									=tf.convert_to_tensor(_UpperCAmelCase  )
        if text_pair is not None:
           if text.shape.rank > 1:
              raise ValueError("""text argument should not be multidimensional when a text pair is supplied!"""  )
           if text_pair.shape.rank > 1:
              raise ValueError("""text_pair should not be multidimensional!"""  )
        if text.shape.rank == 2:
           lowerCamelCase				,       lowerCamelCase									=text[:, 0], text[:, 1]
        lowerCamelCase									=self.unpaired_tokenize(_UpperCAmelCase  )
        if text_pair is None:  # Unpaired text
           if truncation:
              lowerCamelCase									=text[:, : max_length - 2]  # Allow room for special tokens
           lowerCamelCase				,       lowerCamelCase									=combine_segments(
               (text,)							, start_of_sequence_id=self.cls_token_id							, end_of_segment_id=self.sep_token_id  )
        else:  # Paired text
           lowerCamelCase									=self.unpaired_tokenize(_UpperCAmelCase  )
           if truncation:
              lowerCamelCase				,       lowerCamelCase									=self.paired_trimmer.trim([text, text_pair]  )
           lowerCamelCase				,       lowerCamelCase									=combine_segments(
               (text, text_pair)							, start_of_sequence_id=self.cls_token_id							, end_of_segment_id=self.sep_token_id  )
        if padding == "longest":
           lowerCamelCase									=input_ids.bounding_shape(axis=1  )
           if pad_to_multiple_of is not None:
              # No ceiling division in tensorflow, so we negate floordiv instead
              lowerCamelCase									=pad_to_multiple_of * (-tf.math.floordiv(-pad_length							, _UpperCAmelCase  ))
        else:
           lowerCamelCase									=max_length

        lowerCamelCase				,       lowerCamelCase									=pad_model_inputs(_UpperCAmelCase							, max_seq_length=_UpperCAmelCase							, pad_value=self.pad_token_id  )
        lowerCamelCase									={"""input_ids""": input_ids}
        if return_attention_mask:
           lowerCamelCase									=attention_mask
        if return_token_type_ids:
           lowerCamelCase				,       lowerCamelCase									=pad_model_inputs(
               _UpperCAmelCase							, max_seq_length=_UpperCAmelCase							, pad_value=self.pad_token_id  )
           lowerCamelCase									=token_type_ids
        return output





     def 						lowerCamelCase							(	self  )		->				int:
        return {
            "vocab_list": self.vocab_list,
            "do_lower_case": self.do_lower_case,
            "cls_token_id": self.cls_token_id,
            "sep_token_id": self.sep_token_id,
            "pad_token_id": self.pad_token_id,
        }







import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
   import torch

if is_vision_available():
   from PIL import Image

   from transformers import SwinaSRImageProcessor
   from transformers.image_transforms import get_image_size


class 		lowerCAmelCase_				(  unittest.TestCase ):





     def __init__(	self							, _UpperCAmelCase							, _UpperCAmelCase=7							, _UpperCAmelCase=3							, _UpperCAmelCase=18							, _UpperCAmelCase=30							, _UpperCAmelCase=400							, _UpperCAmelCase=True							, _UpperCAmelCase=1 / 255							, _UpperCAmelCase=True							, _UpperCAmelCase=8							, )		->				Tuple:
        lowerCamelCase									=parent
        lowerCamelCase									=batch_size
        lowerCamelCase									=num_channels
        lowerCamelCase									=image_size
        lowerCamelCase									=min_resolution
        lowerCamelCase									=max_resolution
        lowerCamelCase									=do_rescale
        lowerCamelCase									=rescale_factor
        lowerCamelCase									=do_pad
        lowerCamelCase									=pad_size





     def 						lowerCamelCase							(	self  )		->				Optional[Any]:
        return {
            "do_rescale": self.do_rescale,
            "rescale_factor": self.rescale_factor,
            "do_pad": self.do_pad,
            "pad_size": self.pad_size,
        }





     def 						lowerCamelCase							(	self							, _UpperCAmelCase=False							, _UpperCAmelCase=False							, _UpperCAmelCase=False  )		->				int:
        assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

        if equal_resolution:
           lowerCamelCase									=[]
           for i in range(self.batch_size  ):
              image_inputs.append(
                  np.random.randint(
                      255							, size=(self.num_channels, self.max_resolution, self.max_resolution)							, dtype=np.uinta  )  )
        else:
           lowerCamelCase									=[]
           for i in range(self.batch_size  ):
              lowerCamelCase				,       lowerCamelCase									=np.random.choice(np.arange(self.min_resolution							, self.max_resolution  )							, 2  )
              image_inputs.append(np.random.randint(255							, size=(self.num_channels, width, height)							, dtype=np.uinta  )  )

        if not numpify and not torchify:
           # PIL expects the channel dimension as last dimension
           lowerCamelCase									=[Image.fromarray(np.moveaxis(_UpperCAmelCase							, 0							, -1  )  ) for x in image_inputs]

        if torchify:
           lowerCamelCase									=[torch.from_numpy(_UpperCAmelCase  ) for x in image_inputs]

        return image_inputs




@require_torch
@require_vision
class 		lowerCAmelCase_				(  UpperCamelCase				,     unittest.TestCase ):
     __lowerCamelCase       :    Optional[Any] 	=     SwinaSRImageProcessor if is_vision_available() else None





     def 						lowerCamelCase							(	self  )		->				Dict:
        lowerCamelCase									=SwinaSRImageProcessingTester(self  )





     @property
     def 						lowerCamelCase							(	self  )		->				Any:
        return self.image_processor_tester.prepare_image_processor_dict()





     def 						lowerCamelCase							(	self  )		->				Any:
        lowerCamelCase									=self.image_processing_class(**self.image_processor_dict  )
        self.assertTrue(hasattr(_UpperCAmelCase							, """do_rescale"""  )  )
        self.assertTrue(hasattr(_UpperCAmelCase							, """rescale_factor"""  )  )
        self.assertTrue(hasattr(_UpperCAmelCase							, """do_pad"""  )  )
        self.assertTrue(hasattr(_UpperCAmelCase							, """pad_size"""  )  )





     def 						lowerCamelCase							(	self  )		->				Any:
        pass





     def 						lowerCamelCase							(	self							, _UpperCAmelCase  )		->				int:
        lowerCamelCase				,       lowerCamelCase									=get_image_size(_UpperCAmelCase  )
        lowerCamelCase									=self.image_processor_tester.pad_size

        lowerCamelCase									=(old_height // size + 1) * size - old_height
        lowerCamelCase									=(old_width // size + 1) * size - old_width
        return old_height + pad_height, old_width + pad_width





     def 						lowerCamelCase							(	self  )		->				Optional[Any]:
        # Initialize image_processor
        lowerCamelCase									=self.image_processing_class(**self.image_processor_dict  )
        # create random PIL images
        lowerCamelCase									=self.image_processor_tester.prepare_inputs(equal_resolution=_UpperCAmelCase  )
        for image in image_inputs:
           self.assertIsInstance(_UpperCAmelCase							, Image.Image  )

        # Test not batched input
        lowerCamelCase									=image_processor(image_inputs[0]							, return_tensors="""pt"""  ).pixel_values
        lowerCamelCase				,       lowerCamelCase									=self.calculate_expected_size(np.array(image_inputs[0]  )  )
        self.assertEqual(
            encoded_images.shape							, (
                1,
                self.image_processor_tester.num_channels,
                expected_height,
                expected_width,
            )							, )





     def 						lowerCamelCase							(	self  )		->				Any:
        # Initialize image_processor
        lowerCamelCase									=self.image_processing_class(**self.image_processor_dict  )
        # create random numpy tensors
        lowerCamelCase									=self.image_processor_tester.prepare_inputs(equal_resolution=_UpperCAmelCase							, numpify=_UpperCAmelCase  )
        for image in image_inputs:
           self.assertIsInstance(_UpperCAmelCase							, np.ndarray  )

        # Test not batched input
        lowerCamelCase									=image_processor(image_inputs[0]							, return_tensors="""pt"""  ).pixel_values
        lowerCamelCase				,       lowerCamelCase									=self.calculate_expected_size(image_inputs[0]  )
        self.assertEqual(
            encoded_images.shape							, (
                1,
                self.image_processor_tester.num_channels,
                expected_height,
                expected_width,
            )							, )





     def 						lowerCamelCase							(	self  )		->				Dict:
        # Initialize image_processor
        lowerCamelCase									=self.image_processing_class(**self.image_processor_dict  )
        # create random PyTorch tensors
        lowerCamelCase									=self.image_processor_tester.prepare_inputs(equal_resolution=_UpperCAmelCase							, torchify=_UpperCAmelCase  )
        for image in image_inputs:
           self.assertIsInstance(_UpperCAmelCase							, torch.Tensor  )

        # Test not batched input
        lowerCamelCase									=image_processor(image_inputs[0]							, return_tensors="""pt"""  ).pixel_values
        lowerCamelCase				,       lowerCamelCase									=self.calculate_expected_size(image_inputs[0]  )
        self.assertEqual(
            encoded_images.shape							, (
                1,
                self.image_processor_tester.num_channels,
                expected_height,
                expected_width,
            )							, )