




'''simple docstring'''







import inspect
import unittest
from math import floor

from transformers import CvtConfig
from transformers.file_utils import cached_property, is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_vision, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import CvtForImageClassification, CvtModel
   from transformers.models.cvt.modeling_cvt import CVT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
   from PIL import Image

   from transformers import AutoImageProcessor


class 			_a  (	__a ):
       def     A      (	self						:	Any				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.config_class(**self.inputs_dict				)
           self.parent.assertTrue(hasattr(lowercase   ,				'''embed_dim'''				)				)
           self.parent.assertTrue(hasattr(lowercase   ,				'''num_heads'''				)				)






class 			_a  :
       def __init__(	self						:	Union[str, Any]   ,				lowercase						:	Dict   ,				lowercase						:	Dict=13   ,				lowercase						:	List[str]=64   ,				lowercase						:	Optional[Any]=3   ,				lowercase						:	Union[str, Any]=[16, 48, 96]   ,				lowercase						:	Optional[Any]=[1, 3, 6]   ,				lowercase						:	List[Any]=[1, 2, 10]   ,				lowercase						:	Any=[7, 3, 3]   ,				lowercase						:	int=[4, 2, 2]   ,				lowercase						:	Tuple=[2, 1, 1]   ,				lowercase						:	Optional[int]=[2, 2, 2]   ,				lowercase						:	int=[False, False, True]   ,				lowercase						:	str=[0.0, 0.0, 0.0]   ,				lowercase						:	Union[str, Any]=0.02   ,				lowercase						:	Optional[int]=1E-12   ,				lowercase						:	Any=True   ,				lowercase						:	Optional[int]=True   ,				lowercase						:	Tuple=2   ,				):



           '''simple docstring'''

           UpperCAmelCase			       =		parent
           UpperCAmelCase			       =		batch_size
           UpperCAmelCase			       =		image_size
           UpperCAmelCase			       =		patch_sizes
           UpperCAmelCase			       =		patch_stride
           UpperCAmelCase			       =		patch_padding
           UpperCAmelCase			       =		is_training
           UpperCAmelCase			       =		use_labels
           UpperCAmelCase			       =		num_labels
           UpperCAmelCase			       =		num_channels
           UpperCAmelCase			       =		embed_dim
           UpperCAmelCase			       =		num_heads
           UpperCAmelCase			       =		stride_kv
           UpperCAmelCase			       =		depth
           UpperCAmelCase			       =		cls_token
           UpperCAmelCase			       =		attention_drop_rate
           UpperCAmelCase			       =		initializer_range
           UpperCAmelCase			       =		layer_norm_eps
       def     A      (	self						:	Tuple				):



           '''simple docstring'''

           UpperCAmelCase			       =		floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]				)

           UpperCAmelCase			       =		None
           if self.use_labels:
               UpperCAmelCase			       =		ids_tensor([self.batch_size]   ,				self.num_labels				)

           UpperCAmelCase			       =		self.get_config()
           return config, pixel_values, labels
       def     A      (	self						:	Optional[int]				):



           '''simple docstring'''

           return CvtConfig(
               image_size=self.image_size   ,				num_labels=self.num_labels   ,				num_channels=self.num_channels   ,				embed_dim=self.embed_dim   ,				num_heads=self.num_heads   ,				patch_sizes=self.patch_sizes   ,				patch_padding=self.patch_padding   ,				patch_stride=self.patch_stride   ,				stride_kv=self.stride_kv   ,				depth=self.depth   ,				cls_token=self.cls_token   ,				attention_drop_rate=self.attention_drop_rate   ,				initializer_range=self.initializer_range   ,				)
       def     A      (	self						:	Optional[int]   ,				lowercase						:	str   ,				lowercase						:	Tuple   ,				lowercase						:	str				):



           '''simple docstring'''

           UpperCAmelCase			       =		CvtModel(config=lowercase				)
           model.to(lowercase				)
           model.eval()
           UpperCAmelCase			       =		model(lowercase				)
           UpperCAmelCase			       =		(self.image_size, self.image_size)
           UpperCAmelCase       ,     UpperCAmelCase			       =		image_size[0], image_size[1]
           for i in range(len(self.depth				)				):
               UpperCAmelCase			       =		floor(((height + 2 * self.patch_padding[i] - self.patch_sizes[i]) / self.patch_stride[i]) + 1				)
               UpperCAmelCase			       =		floor(((width + 2 * self.patch_padding[i] - self.patch_sizes[i]) / self.patch_stride[i]) + 1				)
           self.parent.assertEqual(result.last_hidden_state.shape   ,				(self.batch_size, self.embed_dim[-1], height, width)				)
       def     A      (	self						:	Dict   ,				lowercase						:	List[str]   ,				lowercase						:	Dict   ,				lowercase						:	int				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.num_labels
           UpperCAmelCase			       =		CvtForImageClassification(lowercase				)
           model.to(lowercase				)
           model.eval()
           UpperCAmelCase			       =		model(lowercase   ,				labels=lowercase				)
           self.parent.assertEqual(result.logits.shape   ,				(self.batch_size, self.num_labels)				)







       def     A      (	self						:	Any				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.prepare_config_and_inputs()
           UpperCAmelCase       ,     UpperCAmelCase       ,     UpperCAmelCase			       =		config_and_inputs
           UpperCAmelCase			       =		{'''pixel_values''': pixel_values}
           return config, inputs_dict






@require_torch
class 			_a  (	__a							,							__a							,							unittest.TestCase ):
       __a				:     Dict   				=					(CvtModel, CvtForImageClassification) if is_torch_available() else ()
       __a				:     Optional[int]   				=					(
           {"""feature-extraction""": CvtModel, """image-classification""": CvtForImageClassification}
           if is_torch_available()
           else {}
       )

       __a				:     List[str]   				=					False
       __a				:     Any   				=					False
       __a				:     Optional[int]   				=					False
       __a				:     Any   				=					False
       __a				:     str   				=					False
       def     A      (	self						:	Tuple				):



           '''simple docstring'''

           UpperCAmelCase			       =		CvtModelTester(self				)
           UpperCAmelCase			       =		ConfigTester(self   ,				config_class=lowercase   ,				has_text_modality=lowercase   ,				hidden_size=37				)
       def     A      (	self						:	str				):



           '''simple docstring'''

           self.create_and_test_config_common_properties()
           self.config_tester.create_and_test_config_to_json_string()
           self.config_tester.create_and_test_config_to_json_file()
           self.config_tester.create_and_test_config_from_and_save_pretrained()
           self.config_tester.create_and_test_config_with_num_labels()
           self.config_tester.check_config_can_be_init_without_params()
           self.config_tester.check_config_arguments_init()
       def     A      (	self						:	Optional[int]				):



           '''simple docstring'''

           return
       @unittest.skip(reason='''Cvt does not output attentions'''				)
       def     A      (	self						:	str				):



           '''simple docstring'''

           pass
       @unittest.skip(reason='''Cvt does not use inputs_embeds'''				)
       def     A      (	self						:	str				):



           '''simple docstring'''

           pass
       @unittest.skip(reason='''Cvt does not support input and output embeddings'''				)
       def     A      (	self						:	Dict				):



           '''simple docstring'''

           pass
       def     A      (	self						:	List[str]				):



           '''simple docstring'''

           UpperCAmelCase       ,     UpperCAmelCase			       =		self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
               UpperCAmelCase			       =		model_class(lowercase				)
               UpperCAmelCase			       =		inspect.signature(model.forward				)
               # signature.parameters is an OrderedDict => so arg_names order is deterministic
               UpperCAmelCase			       =		[*signature.parameters.keys()]

               UpperCAmelCase			       =		['''pixel_values''']
               self.assertListEqual(arg_names[:1]   ,				lowercase				)
       def     A      (	self						:	Optional[Any]				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*lowercase				)
       def     A      (	self						:	Optional[Any]				):



           '''simple docstring'''

           def check_hidden_states_output(lowercase						:	List[str]   ,				lowercase						:	Any   ,				lowercase						:	Dict				):
               UpperCAmelCase			       =		model_class(lowercase				)
               model.to(lowercase				)
               model.eval()

               with torch.no_grad():
                   UpperCAmelCase			       =		model(**self._prepare_for_class(lowercase   ,				lowercase				)				)

               UpperCAmelCase			       =		outputs.hidden_states

               UpperCAmelCase			       =		len(self.model_tester.depth				)
               self.assertEqual(len(lowercase				)   ,				lowercase				)

               # verify the first hidden states (first block)
               self.assertListEqual(
                   list(hidden_states[0].shape[-3:]				)   ,				[
                       self.model_tester.embed_dim[0],
                       self.model_tester.image_size // 4,
                       self.model_tester.image_size // 4,
                   ]   ,				)

           UpperCAmelCase       ,     UpperCAmelCase			       =		self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
               UpperCAmelCase			       =		True
               check_hidden_states_output(lowercase   ,				lowercase   ,				lowercase				)

               # check that output_hidden_states also work using config
               del inputs_dict["output_hidden_states"]
               UpperCAmelCase			       =		True

               check_hidden_states_output(lowercase   ,				lowercase   ,				lowercase				)
       def     A      (	self						:	Tuple				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_image_classification(*lowercase				)
       @unittest.skip('''Will be fixed soon by reducing the size of the model used for common tests.'''				)
       def     A      (	self						:	List[str]				):



           '''simple docstring'''

           pass







       @slow
       def     A      (	self						:	Optional[Any]				):



           '''simple docstring'''

           for model_name in CVT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               UpperCAmelCase			       =		CvtModel.from_pretrained(lowercase				)
               self.assertIsNotNone(lowercase				)






def 						snake_case_							():
    UpperCAmelCase			       =		Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''     )
    return image







@require_torch
@require_vision
class 			_a  (	unittest.TestCase ):
       @cached_property
       def     A      (	self						:	Tuple				):



           '''simple docstring'''

           return AutoImageProcessor.from_pretrained(CVT_PRETRAINED_MODEL_ARCHIVE_LIST[0]				)







       @slow
       def     A      (	self						:	Any				):



           '''simple docstring'''

           UpperCAmelCase			       =		CvtForImageClassification.from_pretrained(CVT_PRETRAINED_MODEL_ARCHIVE_LIST[0]				).to(lowercase				)

           UpperCAmelCase			       =		self.default_image_processor
           UpperCAmelCase			       =		prepare_img()
           UpperCAmelCase			       =		image_processor(images=lowercase   ,				return_tensors='''pt'''				).to(lowercase				)

           # forward pass
           with torch.no_grad():
               UpperCAmelCase			       =		model(**lowercase				)

           # verify the logits
           UpperCAmelCase			       =		torch.Size((1, 1_000)				)
           self.assertEqual(outputs.logits.shape   ,				lowercase				)

           UpperCAmelCase			       =		torch.tensor([0.9285, 0.9015, -0.3150]				).to(lowercase				)

           self.assertTrue(torch.allclose(outputs.logits[0, :3]   ,				lowercase   ,				atol=1E-4				)				)










'''simple docstring'''







import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, PerceiverTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
   A     ='pt'
elif is_tf_available():
   A     ='tf'
else:
   A     ='jax'


class 			_a  (	__a							,							unittest.TestCase ):
       __a				:     Optional[Any]   				=					PerceiverTokenizer
       __a				:     str   				=					False
       def     A      (	self						:	Union[str, Any]				):



           '''simple docstring'''

           super().setUp()
           UpperCAmelCase			       =		PerceiverTokenizer()
           tokenizer.save_pretrained(self.tmpdirname				)
       @cached_property
       def     A      (	self						:	Optional[int]				):



           '''simple docstring'''

           return PerceiverTokenizer.from_pretrained('''deepmind/language-perceiver'''				)
       def     A      (	self						:	Union[str, Any]   ,				**lowercase						:	int				):



           '''simple docstring'''

           return self.tokenizer_class.from_pretrained(self.tmpdirname   ,				**lowercase				)
       def     A      (	self						:	Tuple   ,				lowercase						:	str   ,				lowercase						:	List[str]=False   ,				lowercase						:	Union[str, Any]=20   ,				lowercase						:	Union[str, Any]=5				):



           '''simple docstring'''

           UpperCAmelCase			       =		[]
           for i in range(len(lowercase				)				):
               try:
                   UpperCAmelCase			       =		tokenizer.decode([i]   ,				clean_up_tokenization_spaces=lowercase				)
               except UnicodeDecodeError:
                   pass
               toks.append((i, tok)				)

           UpperCAmelCase			       =		list(filter(lambda lowercase				: re.match(R'''^[ a-zA-Z]+$'''   ,				t[1]				)   ,				lowercase				)				)
           UpperCAmelCase			       =		list(filter(lambda lowercase				: [t[0]] == tokenizer.encode(t[1]   ,				add_special_tokens=lowercase				)   ,				lowercase				)				)
           if max_length is not None and len(lowercase				) > max_length:
               UpperCAmelCase			       =		toks[:max_length]
           if min_length is not None and len(lowercase				) < min_length and len(lowercase				) > 0:
               while len(lowercase				) < min_length:
                   UpperCAmelCase			       =		toks + toks
        # toks_str = [t[1] for t in toks]
           UpperCAmelCase			       =		[t[0] for t in toks]

           # Ensure consistency
           UpperCAmelCase			       =		tokenizer.decode(lowercase   ,				clean_up_tokenization_spaces=lowercase				)
           if " " not in output_txt and len(lowercase				) > 1:
               UpperCAmelCase			       =		(
                   tokenizer.decode([toks_ids[0]]   ,				clean_up_tokenization_spaces=lowercase				)
                   + ''' '''
                   + tokenizer.decode(toks_ids[1:]   ,				clean_up_tokenization_spaces=lowercase				)
               )
           if with_prefix_space:
               UpperCAmelCase			       =		''' ''' + output_txt
           UpperCAmelCase			       =		tokenizer.encode(lowercase   ,				add_special_tokens=lowercase				)
           return output_txt, output_ids
       def     A      (	self						:	Optional[int]				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.perceiver_tokenizer
           UpperCAmelCase			       =		'''Unicode €.'''
           UpperCAmelCase			       =		tokenizer(lowercase				)
           UpperCAmelCase			       =		[4, 91, 116, 111, 105, 117, 106, 107, 38, 232, 136, 178, 52, 5]
           self.assertEqual(encoded['''input_ids''']   ,				lowercase				)

           # decoding
           UpperCAmelCase			       =		tokenizer.decode(lowercase				)
           self.assertEqual(lowercase   ,				'''[CLS]Unicode €.[SEP]'''				)

           UpperCAmelCase			       =		tokenizer('''e è é ê ë'''				)
           UpperCAmelCase			       =		[4, 107, 38, 201, 174, 38, 201, 175, 38, 201, 176, 38, 201, 177, 5]
           self.assertEqual(encoded['''input_ids''']   ,				lowercase				)
           # decoding
           UpperCAmelCase			       =		tokenizer.decode(lowercase				)
           self.assertEqual(lowercase   ,				'''[CLS]e è é ê ë[SEP]'''				)

           # encode/decode, but with `encode` instead of `__call__`
           self.assertEqual(tokenizer.decode(tokenizer.encode('''e è é ê ë'''				)				)   ,				'''[CLS]e è é ê ë[SEP]'''				)
       def     A      (	self						:	str				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.perceiver_tokenizer
           UpperCAmelCase			       =		['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
           # fmt: off
           UpperCAmelCase			       =		[4, 71, 38, 114, 117, 116, 109, 38, 118, 103, 120, 103, 109, 120, 103, 118, 110, 38, 108, 117, 120, 38, 121, 123, 115, 115, 103, 120, 111, 128, 103, 122, 111, 117, 116, 52, 5, 0]
           # fmt: on
           UpperCAmelCase			       =		tokenizer(lowercase   ,				padding=lowercase   ,				return_tensors=lowercase				)
           self.assertIsInstance(lowercase   ,				lowercase				)

           if FRAMEWORK != "jax":
               UpperCAmelCase			       =		list(batch.input_ids.numpy()[0]				)
           else:
               UpperCAmelCase			       =		list(batch.input_ids.tolist()[0]				)

           self.assertListEqual(lowercase   ,				lowercase				)

           self.assertEqual((2, 38)   ,				batch.input_ids.shape				)
           self.assertEqual((2, 38)   ,				batch.attention_mask.shape				)
       def     A      (	self						:	str				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.perceiver_tokenizer
           UpperCAmelCase			       =		['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
           UpperCAmelCase			       =		tokenizer(lowercase   ,				padding=lowercase   ,				return_tensors=lowercase				)
           # check if input_ids are returned and no decoder_input_ids
           self.assertIn('''input_ids'''   ,				lowercase				)
           self.assertIn('''attention_mask'''   ,				lowercase				)
           self.assertNotIn('''decoder_input_ids'''   ,				lowercase				)
           self.assertNotIn('''decoder_attention_mask'''   ,				lowercase				)
       def     A      (	self						:	Dict				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.perceiver_tokenizer
           UpperCAmelCase			       =		[
               '''Summary of the text.''',
               '''Another summary.''',
           ]
           UpperCAmelCase			       =		tokenizer(
               text_target=lowercase   ,				max_length=32   ,				padding='''max_length'''   ,				truncation=lowercase   ,				return_tensors=lowercase				)
           self.assertEqual(32   ,				targets['''input_ids'''].shape[1]				)
       def     A      (	self						:	int				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.get_tokenizers()
           for tokenizer in tokenizers:
               with self.subTest(f"{tokenizer.__class__.__name__}"				):
                   self.assertNotEqual(tokenizer.model_max_length   ,				42				)

        # Now let's start the test
           UpperCAmelCase			       =		self.get_tokenizers()
           for tokenizer in tokenizers:
               with self.subTest(f"{tokenizer.__class__.__name__}"				):
                   # Isolate this from the other tests because we save additional tokens/etc
                   UpperCAmelCase			       =		tempfile.mkdtemp()

                   UpperCAmelCase			       =		''' He is very happy, UNwant\u00E9d,running'''
                   UpperCAmelCase			       =		tokenizer.encode(lowercase   ,				add_special_tokens=lowercase				)
                   tokenizer.save_pretrained(lowercase				)

                   UpperCAmelCase			       =		tokenizer.__class__.from_pretrained(lowercase				)
                   UpperCAmelCase			       =		after_tokenizer.encode(lowercase   ,				add_special_tokens=lowercase				)
                   self.assertListEqual(lowercase   ,				lowercase				)

                   shutil.rmtree(lowercase				)

           UpperCAmelCase			       =		self.get_tokenizers(model_max_length=42				)
           for tokenizer in tokenizers:
               with self.subTest(f"{tokenizer.__class__.__name__}"				):
                   # Isolate this from the other tests because we save additional tokens/etc
                   UpperCAmelCase			       =		tempfile.mkdtemp()

                   UpperCAmelCase			       =		''' He is very happy, UNwant\u00E9d,running'''
                   tokenizer.add_tokens(['''bim''', '''bambam''']				)
                   UpperCAmelCase			       =		tokenizer.additional_special_tokens
                   additional_special_tokens.append('''new_additional_special_token'''				)
                   tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}				)
                   UpperCAmelCase			       =		tokenizer.encode(lowercase   ,				add_special_tokens=lowercase				)
                   tokenizer.save_pretrained(lowercase				)

                   UpperCAmelCase			       =		tokenizer.__class__.from_pretrained(lowercase				)
                   UpperCAmelCase			       =		after_tokenizer.encode(lowercase   ,				add_special_tokens=lowercase				)
                   self.assertListEqual(lowercase   ,				lowercase				)
                   self.assertIn('''new_additional_special_token'''   ,				after_tokenizer.additional_special_tokens				)
                   self.assertEqual(after_tokenizer.model_max_length   ,				42				)

                   UpperCAmelCase			       =		tokenizer.__class__.from_pretrained(lowercase   ,				model_max_length=43				)
                   self.assertEqual(tokenizer.model_max_length   ,				43				)

                   shutil.rmtree(lowercase				)
       def     A      (	self						:	Optional[int]				):



           '''simple docstring'''

           UpperCAmelCase			       =		[]
           if self.test_slow_tokenizer:
               tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())				)

           if self.test_rust_tokenizer:
               tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())				)

           for tokenizer_class, tokenizer_utils in tokenizer_list:
               with tempfile.TemporaryDirectory() as tmp_dir:
                   tokenizer_utils.save_pretrained(lowercase				)

                   with open(os.path.join(lowercase   ,				'''special_tokens_map.json'''				)   ,				encoding='''utf-8'''				) as json_file:
                       UpperCAmelCase			       =		json.load(lowercase				)

                   with open(os.path.join(lowercase   ,				'''tokenizer_config.json'''				)   ,				encoding='''utf-8'''				) as json_file:
                       UpperCAmelCase			       =		json.load(lowercase				)

                   UpperCAmelCase			       =		[f"<extra_id_{i}>" for i in range(125				)]

                   UpperCAmelCase			       =		added_tokens_extra_ids + [
                       '''an_additional_special_token'''
                   ]
                   UpperCAmelCase			       =		added_tokens_extra_ids + [
                       '''an_additional_special_token'''
                   ]

                   with open(os.path.join(lowercase   ,				'''special_tokens_map.json'''				)   ,				'''w'''   ,				encoding='''utf-8'''				) as outfile:
                       json.dump(lowercase   ,				lowercase				)
                   with open(os.path.join(lowercase   ,				'''tokenizer_config.json'''				)   ,				'''w'''   ,				encoding='''utf-8'''				) as outfile:
                       json.dump(lowercase   ,				lowercase				)

                   # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                   # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                   # "special_tokens_map.json" files
                   UpperCAmelCase			       =		tokenizer_class.from_pretrained(
                       lowercase   ,				)
                   self.assertIn(
                       '''an_additional_special_token'''   ,				tokenizer_without_change_in_init.additional_special_tokens				)
                   self.assertEqual(
                       ['''an_additional_special_token''']   ,				tokenizer_without_change_in_init.convert_ids_to_tokens(
                           tokenizer_without_change_in_init.convert_tokens_to_ids(['''an_additional_special_token''']				)				)   ,				)

                   # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                   UpperCAmelCase			       =		added_tokens_extra_ids + [AddedToken('''a_new_additional_special_token'''   ,				lstrip=lowercase				)]
                   UpperCAmelCase			       =		tokenizer_class.from_pretrained(
                       lowercase   ,				additional_special_tokens=lowercase   ,				)

                   self.assertIn('''a_new_additional_special_token'''   ,				tokenizer.additional_special_tokens				)
                   self.assertEqual(
                       ['''a_new_additional_special_token''']   ,				tokenizer.convert_ids_to_tokens(
                           tokenizer.convert_tokens_to_ids(['''a_new_additional_special_token''']				)				)   ,				)
       def     A      (	self						:	Optional[int]				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.perceiver_tokenizer
           self.assertEqual(tokenizer.decode([178]				)   ,				'''�'''				)
       def     A      (	self						:	Union[str, Any]				):



           '''simple docstring'''

           pass
       def     A      (	self						:	Any				):



           '''simple docstring'''

           pass
       def     A      (	self						:	Dict				):



           '''simple docstring'''

           pass
       def     A      (	self						:	str				):



           '''simple docstring'''

           pass







       def     A      (	self						:	List[str]				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.get_tokenizers(fast=lowercase   ,				do_lower_case=lowercase				)
           for tokenizer in tokenizers:
               with self.subTest(f"{tokenizer.__class__.__name__}"				):
                   UpperCAmelCase			       =		['''[CLS]''', '''t''', '''h''', '''i''', '''s''', ''' ''', '''i''', '''s''', ''' ''', '''a''', ''' ''', '''t''', '''e''', '''s''', '''t''', '''[SEP]''']
                   UpperCAmelCase			       =		tokenizer.convert_tokens_to_string(lowercase				)

                   self.assertIsInstance(lowercase   ,				lowercase				)




