




'''simple docstring'''







from typing import List, Optional, Tuple, Union

import torch
from torch import nn
from torch.nn import CrossEntropyLoss

from ... import AutoBackbone
from ...modeling_outputs import SemanticSegmenterOutput
from ...modeling_utils import PreTrainedModel
from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, replace_return_docstrings
from ...utils.backbone_utils import BackboneMixin
from .configuration_upernet import UperNetConfig


A     =[
    'openmmlab/upernet-convnext-tiny',
    # See all UperNet models at https://huggingface.co/models?filter=upernet
]

# General docstring
A     ='UperNetConfig'


class 			_a  (	nn.Module ):
       def __init__(	self						:	Union[str, Any]   ,				lowercase						:	int   ,				lowercase						:	int   ,				lowercase						:	Union[int, Tuple[int, int]]   ,				lowercase						:	Union[int, Tuple[int, int], str] = 0   ,				lowercase						:	bool = False   ,				lowercase						:	Union[int, Tuple[int, int]] = 1   ,				):



           '''simple docstring'''

           super().__init__()
           UpperCAmelCase			       =		nn.Convad(
               in_channels=lowercase   ,				out_channels=lowercase   ,				kernel_size=lowercase   ,				padding=lowercase   ,				bias=lowercase   ,				dilation=lowercase   ,				)
           UpperCAmelCase			       =		nn.BatchNormad(lowercase				)
           UpperCAmelCase			       =		nn.ReLU()







       def     A      (	self						:	Dict   ,				lowercase						:	torch.Tensor				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.conv(lowercase				)
           UpperCAmelCase			       =		self.batch_norm(lowercase				)
           UpperCAmelCase			       =		self.activation(lowercase				)

           return output






class 			_a  (	nn.Module ):
       def __init__(	self						:	Dict   ,				lowercase						:	int   ,				lowercase						:	int   ,				lowercase						:	int				):



           '''simple docstring'''

           super().__init__()
           UpperCAmelCase			       =		[
               nn.AdaptiveAvgPoolad(lowercase				),
               UperNetConvModule(lowercase   ,				lowercase   ,				kernel_size=1				),
           ]
           for i, layer in enumerate(self.layers				):
               self.add_module(str(lowercase				)   ,				lowercase				)







       def     A      (	self						:	Any   ,				lowercase						:	torch.Tensor				):



           '''simple docstring'''

           UpperCAmelCase			       =		input
           for layer in self.layers:
               UpperCAmelCase			       =		layer(lowercase				)
           return hidden_state






class 			_a  (	nn.Module ):
       def __init__(	self						:	List[Any]   ,				lowercase						:	Tuple[int, ...]   ,				lowercase						:	int   ,				lowercase						:	int   ,				lowercase						:	bool				):



           '''simple docstring'''

           super().__init__()
           UpperCAmelCase			       =		pool_scales
           UpperCAmelCase			       =		align_corners
           UpperCAmelCase			       =		in_channels
           UpperCAmelCase			       =		channels
           UpperCAmelCase			       =		[]
           for i, pool_scale in enumerate(lowercase				):
               UpperCAmelCase			       =		UperNetPyramidPoolingBlock(pool_scale=lowercase   ,				in_channels=lowercase   ,				channels=lowercase				)
               self.blocks.append(lowercase				)
               self.add_module(str(lowercase				)   ,				lowercase				)







       def     A      (	self						:	Union[str, Any]   ,				lowercase						:	torch.Tensor				):



           '''simple docstring'''

           UpperCAmelCase			       =		[]
           for ppm in self.blocks:
               UpperCAmelCase			       =		ppm(lowercase				)
               UpperCAmelCase			       =		nn.functional.interpolate(
                   lowercase   ,				size=x.size()[2:]   ,				mode='''bilinear'''   ,				align_corners=self.align_corners				)
               ppm_outs.append(lowercase				)
           return ppm_outs






class 			_a  (	nn.Module ):
       def __init__(	self						:	Union[str, Any]   ,				lowercase						:	List[Any]   ,				lowercase						:	Dict				):



           '''simple docstring'''

           super().__init__()

           UpperCAmelCase			       =		config
           UpperCAmelCase			       =		config.pool_scales  # e.g. (1, 2, 3, 6)
           UpperCAmelCase			       =		in_channels
           UpperCAmelCase			       =		config.hidden_size
           UpperCAmelCase			       =		False
           UpperCAmelCase			       =		nn.Convad(self.channels   ,				config.num_labels   ,				kernel_size=1				)

           # PSP Module
           UpperCAmelCase			       =		UperNetPyramidPoolingModule(
               self.pool_scales   ,				self.in_channels[-1]   ,				self.channels   ,				align_corners=self.align_corners   ,				)
           UpperCAmelCase			       =		UperNetConvModule(
               self.in_channels[-1] + len(self.pool_scales				) * self.channels   ,				self.channels   ,				kernel_size=3   ,				padding=1   ,				)
           # FPN Module
           UpperCAmelCase			       =		nn.ModuleList()
           UpperCAmelCase			       =		nn.ModuleList()
           for in_channels in self.in_channels[:-1]:  # skip the top layer
               UpperCAmelCase			       =		UperNetConvModule(lowercase   ,				self.channels   ,				kernel_size=1				)
               UpperCAmelCase			       =		UperNetConvModule(self.channels   ,				self.channels   ,				kernel_size=3   ,				padding=1				)
               self.lateral_convs.append(lowercase				)
               self.fpn_convs.append(lowercase				)

           UpperCAmelCase			       =		UperNetConvModule(
               len(self.in_channels				) * self.channels   ,				self.channels   ,				kernel_size=3   ,				padding=1   ,				)
       def     A      (	self						:	Optional[int]				):



           '''simple docstring'''

           self.apply(self._init_weights				)
       def     A      (	self						:	List[Any]   ,				lowercase						:	Optional[int]				):



           '''simple docstring'''

           if isinstance(lowercase   ,				nn.Convad				):
               module.weight.data.normal_(mean=0.0   ,				std=self.config.initializer_range				)
               if module.bias is not None:
                   module.bias.data.zero_()
       def     A      (	self						:	str   ,				lowercase						:	Optional[Any]				):



           '''simple docstring'''

           UpperCAmelCase			       =		inputs[-1]
           UpperCAmelCase			       =		[x]
           psp_outs.extend(self.psp_modules(lowercase				)				)
           UpperCAmelCase			       =		torch.cat(lowercase   ,				dim=1				)
           UpperCAmelCase			       =		self.bottleneck(lowercase				)

           return output







       def     A      (	self						:	Optional[int]   ,				lowercase						:	torch.Tensor				):



           '''simple docstring'''

           UpperCAmelCase			       =		[lateral_conv(encoder_hidden_states[i]				) for i, lateral_conv in enumerate(self.lateral_convs				)]

           laterals.append(self.psp_forward(lowercase				)				)

           # build top-down path
           UpperCAmelCase			       =		len(lowercase				)
           for i in range(used_backbone_levels - 1   ,				0   ,				-1				):
               UpperCAmelCase			       =		laterals[i - 1].shape[2:]
               UpperCAmelCase			       =		laterals[i - 1] + nn.functional.interpolate(
                   laterals[i]   ,				size=lowercase   ,				mode='''bilinear'''   ,				align_corners=self.align_corners				)

           # build outputs
           UpperCAmelCase			       =		[self.fpn_convs[i](laterals[i]				) for i in range(used_backbone_levels - 1				)]
           # append psp feature
           fpn_outs.append(laterals[-1]				)

           for i in range(used_backbone_levels - 1   ,				0   ,				-1				):
               UpperCAmelCase			       =		nn.functional.interpolate(
                   fpn_outs[i]   ,				size=fpn_outs[0].shape[2:]   ,				mode='''bilinear'''   ,				align_corners=self.align_corners				)
           UpperCAmelCase			       =		torch.cat(lowercase   ,				dim=1				)
           UpperCAmelCase			       =		self.fpn_bottleneck(lowercase				)
           UpperCAmelCase			       =		self.classifier(lowercase				)

           return output






class 			_a  (	nn.Module ):
       def __init__(	self						:	Optional[Any]   ,				lowercase						:	List[Any]   ,				lowercase						:	int = 2   ,				lowercase						:	int = 3   ,				lowercase						:	Union[int, Tuple[int, int]] = 1				):



           '''simple docstring'''

           super().__init__()

           UpperCAmelCase			       =		config
           UpperCAmelCase			       =		config.auxiliary_in_channels
           UpperCAmelCase			       =		config.auxiliary_channels
           UpperCAmelCase			       =		config.auxiliary_num_convs
           UpperCAmelCase			       =		config.auxiliary_concat_input
           UpperCAmelCase			       =		in_index

           UpperCAmelCase			       =		(kernel_size // 2) * dilation
           UpperCAmelCase			       =		[]
           convs.append(
               UperNetConvModule(
                   self.in_channels   ,				self.channels   ,				kernel_size=lowercase   ,				padding=lowercase   ,				dilation=lowercase				)				)
           for i in range(self.num_convs - 1				):
               convs.append(
                   UperNetConvModule(
                       self.channels   ,				self.channels   ,				kernel_size=lowercase   ,				padding=lowercase   ,				dilation=lowercase				)				)
           if self.num_convs == 0:
               UpperCAmelCase			       =		nn.Identity()
           else:
               UpperCAmelCase			       =		nn.Sequential(*lowercase				)
           if self.concat_input:
               UpperCAmelCase			       =		UperNetConvModule(
                   self.in_channels + self.channels   ,				self.channels   ,				kernel_size=lowercase   ,				padding=kernel_size // 2				)

           UpperCAmelCase			       =		nn.Convad(self.channels   ,				config.num_labels   ,				kernel_size=1				)
       def     A      (	self						:	List[str]				):



           '''simple docstring'''

           self.apply(self._init_weights				)
       def     A      (	self						:	List[Any]   ,				lowercase						:	List[str]				):



           '''simple docstring'''

           if isinstance(lowercase   ,				nn.Convad				):
               module.weight.data.normal_(mean=0.0   ,				std=self.config.initializer_range				)
               if module.bias is not None:
                   module.bias.data.zero_()







       def     A      (	self						:	Any   ,				lowercase						:	torch.Tensor				):



           '''simple docstring'''

           UpperCAmelCase			       =		encoder_hidden_states[self.in_index]
           UpperCAmelCase			       =		self.convs(lowercase				)
           if self.concat_input:
               UpperCAmelCase			       =		self.conv_cat(torch.cat([hidden_states, output]   ,				dim=1				)				)
           UpperCAmelCase			       =		self.classifier(lowercase				)
           return output






class 			_a  (	__a ):
       __a				:     int   				=					UperNetConfig
       __a				:     Optional[int]   				=					"""pixel_values"""
       __a				:     Optional[int]   				=					True
       def     A      (	self						:	int   ,				lowercase						:	str				):



           '''simple docstring'''

           if isinstance(lowercase   ,				lowercase				):
               module.backbone.init_weights()
               module.decode_head.init_weights()
               module.auxiliary_head.init_weights()
       def     A      (	self						:	List[Any]				):



           '''simple docstring'''

           self.backbone.init_weights()
           self.decode_head.init_weights()
           self.auxiliary_head.init_weights()







       def     A      (	self						:	int   ,				lowercase						:	Any   ,				lowercase						:	List[str]=False				):



           '''simple docstring'''

           if isinstance(lowercase   ,				lowercase				):
               UpperCAmelCase			       =		value






A     =r'\n    Parameters:\n    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use\n    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and\n    behavior.\n        config ([`UperNetConfig`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.\n'

A     =r'\n    Args:\n        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):\n            Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using\n            [`AutoImageProcessor`]. See [`SegformerImageProcessor.__call__`] for details.\n        output_attentions (`bool`, *optional*):\n            Whether or not to return the attentions tensors of all attention layers in case the backbone has them. See\n            `attentions` under returned tensors for more detail.\n        output_hidden_states (`bool`, *optional*):\n            Whether or not to return the hidden states of all layers of the backbone. See `hidden_states` under\n            returned tensors for more detail.\n        return_dict (`bool`, *optional*):\n            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.\n'


@add_start_docstrings(
    """UperNet framework leveraging any vision backbone e.g. for ADE20k, CityScapes."""							,							__a							,							)
class 			_a  (	__a ):
       def __init__(	self						:	List[Any]   ,				lowercase						:	Dict				):



           '''simple docstring'''

           super().__init__(lowercase				)

           UpperCAmelCase			       =		AutoBackbone.from_config(config.backbone_config				)

           # Semantic segmentation head(s)
           UpperCAmelCase			       =		UperNetHead(lowercase   ,				in_channels=self.backbone.channels				)
           UpperCAmelCase			       =		UperNetFCNHead(lowercase				) if config.use_auxiliary_head else None

           # Initialize weights and apply final processing
           self.post_init()







       @add_start_docstrings_to_model_forward(UPERNET_INPUTS_DOCSTRING.format('''batch_size, sequence_length'''				)				)
       @replace_return_docstrings(output_type=lowercase   ,				config_class=_CONFIG_FOR_DOC				)
       def     A      (	self						:	Any   ,				lowercase						:	Optional[torch.Tensor] = None   ,				lowercase						:	Optional[bool] = None   ,				lowercase						:	Optional[bool] = None   ,				lowercase						:	Optional[torch.Tensor] = None   ,				lowercase						:	Optional[bool] = None   ,				):



           '''simple docstring'''

           UpperCAmelCase			       =		return_dict if return_dict is not None else self.config.use_return_dict
           UpperCAmelCase			       =		(
               output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
           )
           UpperCAmelCase			       =		output_attentions if output_attentions is not None else self.config.output_attentions

           UpperCAmelCase			       =		self.backbone.forward_with_filtered_kwargs(
               lowercase   ,				output_hidden_states=lowercase   ,				output_attentions=lowercase				)
           UpperCAmelCase			       =		outputs.feature_maps

           UpperCAmelCase			       =		self.decode_head(lowercase				)
           UpperCAmelCase			       =		nn.functional.interpolate(lowercase   ,				size=pixel_values.shape[2:]   ,				mode='''bilinear'''   ,				align_corners=lowercase				)

           UpperCAmelCase			       =		None
           if self.auxiliary_head is not None:
               UpperCAmelCase			       =		self.auxiliary_head(lowercase				)
               UpperCAmelCase			       =		nn.functional.interpolate(
                   lowercase   ,				size=pixel_values.shape[2:]   ,				mode='''bilinear'''   ,				align_corners=lowercase				)

           UpperCAmelCase			       =		None
           if labels is not None:
               if self.config.num_labels == 1:
                   raise ValueError('''The number of labels should be greater than one'''				)
               else:
                   # compute weighted loss
                   UpperCAmelCase			       =		CrossEntropyLoss(ignore_index=self.config.loss_ignore_index				)
                   UpperCAmelCase			       =		loss_fct(lowercase   ,				lowercase				)
                   UpperCAmelCase			       =		loss_fct(lowercase   ,				lowercase				)
                   UpperCAmelCase			       =		main_loss + self.config.auxiliary_loss_weight * auxiliary_loss

           if not return_dict:
               if output_hidden_states:
                   UpperCAmelCase			       =		(logits,) + outputs[1:]
               else:
                   UpperCAmelCase			       =		(logits,) + outputs[2:]
               return ((loss,) + output) if loss is not None else output

           return SemanticSegmenterOutput(
               loss=lowercase   ,				logits=lowercase   ,				hidden_states=outputs.hidden_states   ,				attentions=outputs.attentions   ,				)










'''simple docstring'''







import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, PerceiverTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
   A     ='pt'
elif is_tf_available():
   A     ='tf'
else:
   A     ='jax'


class 			_a  (	__a							,							unittest.TestCase ):
       __a				:     Optional[Any]   				=					PerceiverTokenizer
       __a				:     str   				=					False
       def     A      (	self						:	Union[str, Any]				):



           '''simple docstring'''

           super().setUp()
           UpperCAmelCase			       =		PerceiverTokenizer()
           tokenizer.save_pretrained(self.tmpdirname				)
       @cached_property
       def     A      (	self						:	Optional[int]				):



           '''simple docstring'''

           return PerceiverTokenizer.from_pretrained('''deepmind/language-perceiver'''				)
       def     A      (	self						:	Union[str, Any]   ,				**lowercase						:	int				):



           '''simple docstring'''

           return self.tokenizer_class.from_pretrained(self.tmpdirname   ,				**lowercase				)
       def     A      (	self						:	Tuple   ,				lowercase						:	str   ,				lowercase						:	List[str]=False   ,				lowercase						:	Union[str, Any]=20   ,				lowercase						:	Union[str, Any]=5				):



           '''simple docstring'''

           UpperCAmelCase			       =		[]
           for i in range(len(lowercase				)				):
               try:
                   UpperCAmelCase			       =		tokenizer.decode([i]   ,				clean_up_tokenization_spaces=lowercase				)
               except UnicodeDecodeError:
                   pass
               toks.append((i, tok)				)

           UpperCAmelCase			       =		list(filter(lambda lowercase				: re.match(R'''^[ a-zA-Z]+$'''   ,				t[1]				)   ,				lowercase				)				)
           UpperCAmelCase			       =		list(filter(lambda lowercase				: [t[0]] == tokenizer.encode(t[1]   ,				add_special_tokens=lowercase				)   ,				lowercase				)				)
           if max_length is not None and len(lowercase				) > max_length:
               UpperCAmelCase			       =		toks[:max_length]
           if min_length is not None and len(lowercase				) < min_length and len(lowercase				) > 0:
               while len(lowercase				) < min_length:
                   UpperCAmelCase			       =		toks + toks
        # toks_str = [t[1] for t in toks]
           UpperCAmelCase			       =		[t[0] for t in toks]

           # Ensure consistency
           UpperCAmelCase			       =		tokenizer.decode(lowercase   ,				clean_up_tokenization_spaces=lowercase				)
           if " " not in output_txt and len(lowercase				) > 1:
               UpperCAmelCase			       =		(
                   tokenizer.decode([toks_ids[0]]   ,				clean_up_tokenization_spaces=lowercase				)
                   + ''' '''
                   + tokenizer.decode(toks_ids[1:]   ,				clean_up_tokenization_spaces=lowercase				)
               )
           if with_prefix_space:
               UpperCAmelCase			       =		''' ''' + output_txt
           UpperCAmelCase			       =		tokenizer.encode(lowercase   ,				add_special_tokens=lowercase				)
           return output_txt, output_ids
       def     A      (	self						:	Optional[int]				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.perceiver_tokenizer
           UpperCAmelCase			       =		'''Unicode €.'''
           UpperCAmelCase			       =		tokenizer(lowercase				)
           UpperCAmelCase			       =		[4, 91, 116, 111, 105, 117, 106, 107, 38, 232, 136, 178, 52, 5]
           self.assertEqual(encoded['''input_ids''']   ,				lowercase				)

           # decoding
           UpperCAmelCase			       =		tokenizer.decode(lowercase				)
           self.assertEqual(lowercase   ,				'''[CLS]Unicode €.[SEP]'''				)

           UpperCAmelCase			       =		tokenizer('''e è é ê ë'''				)
           UpperCAmelCase			       =		[4, 107, 38, 201, 174, 38, 201, 175, 38, 201, 176, 38, 201, 177, 5]
           self.assertEqual(encoded['''input_ids''']   ,				lowercase				)
           # decoding
           UpperCAmelCase			       =		tokenizer.decode(lowercase				)
           self.assertEqual(lowercase   ,				'''[CLS]e è é ê ë[SEP]'''				)

           # encode/decode, but with `encode` instead of `__call__`
           self.assertEqual(tokenizer.decode(tokenizer.encode('''e è é ê ë'''				)				)   ,				'''[CLS]e è é ê ë[SEP]'''				)
       def     A      (	self						:	str				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.perceiver_tokenizer
           UpperCAmelCase			       =		['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
           # fmt: off
           UpperCAmelCase			       =		[4, 71, 38, 114, 117, 116, 109, 38, 118, 103, 120, 103, 109, 120, 103, 118, 110, 38, 108, 117, 120, 38, 121, 123, 115, 115, 103, 120, 111, 128, 103, 122, 111, 117, 116, 52, 5, 0]
           # fmt: on
           UpperCAmelCase			       =		tokenizer(lowercase   ,				padding=lowercase   ,				return_tensors=lowercase				)
           self.assertIsInstance(lowercase   ,				lowercase				)

           if FRAMEWORK != "jax":
               UpperCAmelCase			       =		list(batch.input_ids.numpy()[0]				)
           else:
               UpperCAmelCase			       =		list(batch.input_ids.tolist()[0]				)

           self.assertListEqual(lowercase   ,				lowercase				)

           self.assertEqual((2, 38)   ,				batch.input_ids.shape				)
           self.assertEqual((2, 38)   ,				batch.attention_mask.shape				)
       def     A      (	self						:	str				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.perceiver_tokenizer
           UpperCAmelCase			       =		['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
           UpperCAmelCase			       =		tokenizer(lowercase   ,				padding=lowercase   ,				return_tensors=lowercase				)
           # check if input_ids are returned and no decoder_input_ids
           self.assertIn('''input_ids'''   ,				lowercase				)
           self.assertIn('''attention_mask'''   ,				lowercase				)
           self.assertNotIn('''decoder_input_ids'''   ,				lowercase				)
           self.assertNotIn('''decoder_attention_mask'''   ,				lowercase				)
       def     A      (	self						:	Dict				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.perceiver_tokenizer
           UpperCAmelCase			       =		[
               '''Summary of the text.''',
               '''Another summary.''',
           ]
           UpperCAmelCase			       =		tokenizer(
               text_target=lowercase   ,				max_length=32   ,				padding='''max_length'''   ,				truncation=lowercase   ,				return_tensors=lowercase				)
           self.assertEqual(32   ,				targets['''input_ids'''].shape[1]				)
       def     A      (	self						:	int				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.get_tokenizers()
           for tokenizer in tokenizers:
               with self.subTest(f"{tokenizer.__class__.__name__}"				):
                   self.assertNotEqual(tokenizer.model_max_length   ,				42				)

        # Now let's start the test
           UpperCAmelCase			       =		self.get_tokenizers()
           for tokenizer in tokenizers:
               with self.subTest(f"{tokenizer.__class__.__name__}"				):
                   # Isolate this from the other tests because we save additional tokens/etc
                   UpperCAmelCase			       =		tempfile.mkdtemp()

                   UpperCAmelCase			       =		''' He is very happy, UNwant\u00E9d,running'''
                   UpperCAmelCase			       =		tokenizer.encode(lowercase   ,				add_special_tokens=lowercase				)
                   tokenizer.save_pretrained(lowercase				)

                   UpperCAmelCase			       =		tokenizer.__class__.from_pretrained(lowercase				)
                   UpperCAmelCase			       =		after_tokenizer.encode(lowercase   ,				add_special_tokens=lowercase				)
                   self.assertListEqual(lowercase   ,				lowercase				)

                   shutil.rmtree(lowercase				)

           UpperCAmelCase			       =		self.get_tokenizers(model_max_length=42				)
           for tokenizer in tokenizers:
               with self.subTest(f"{tokenizer.__class__.__name__}"				):
                   # Isolate this from the other tests because we save additional tokens/etc
                   UpperCAmelCase			       =		tempfile.mkdtemp()

                   UpperCAmelCase			       =		''' He is very happy, UNwant\u00E9d,running'''
                   tokenizer.add_tokens(['''bim''', '''bambam''']				)
                   UpperCAmelCase			       =		tokenizer.additional_special_tokens
                   additional_special_tokens.append('''new_additional_special_token'''				)
                   tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}				)
                   UpperCAmelCase			       =		tokenizer.encode(lowercase   ,				add_special_tokens=lowercase				)
                   tokenizer.save_pretrained(lowercase				)

                   UpperCAmelCase			       =		tokenizer.__class__.from_pretrained(lowercase				)
                   UpperCAmelCase			       =		after_tokenizer.encode(lowercase   ,				add_special_tokens=lowercase				)
                   self.assertListEqual(lowercase   ,				lowercase				)
                   self.assertIn('''new_additional_special_token'''   ,				after_tokenizer.additional_special_tokens				)
                   self.assertEqual(after_tokenizer.model_max_length   ,				42				)

                   UpperCAmelCase			       =		tokenizer.__class__.from_pretrained(lowercase   ,				model_max_length=43				)
                   self.assertEqual(tokenizer.model_max_length   ,				43				)

                   shutil.rmtree(lowercase				)
       def     A      (	self						:	Optional[int]				):



           '''simple docstring'''

           UpperCAmelCase			       =		[]
           if self.test_slow_tokenizer:
               tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())				)

           if self.test_rust_tokenizer:
               tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())				)

           for tokenizer_class, tokenizer_utils in tokenizer_list:
               with tempfile.TemporaryDirectory() as tmp_dir:
                   tokenizer_utils.save_pretrained(lowercase				)

                   with open(os.path.join(lowercase   ,				'''special_tokens_map.json'''				)   ,				encoding='''utf-8'''				) as json_file:
                       UpperCAmelCase			       =		json.load(lowercase				)

                   with open(os.path.join(lowercase   ,				'''tokenizer_config.json'''				)   ,				encoding='''utf-8'''				) as json_file:
                       UpperCAmelCase			       =		json.load(lowercase				)

                   UpperCAmelCase			       =		[f"<extra_id_{i}>" for i in range(125				)]

                   UpperCAmelCase			       =		added_tokens_extra_ids + [
                       '''an_additional_special_token'''
                   ]
                   UpperCAmelCase			       =		added_tokens_extra_ids + [
                       '''an_additional_special_token'''
                   ]

                   with open(os.path.join(lowercase   ,				'''special_tokens_map.json'''				)   ,				'''w'''   ,				encoding='''utf-8'''				) as outfile:
                       json.dump(lowercase   ,				lowercase				)
                   with open(os.path.join(lowercase   ,				'''tokenizer_config.json'''				)   ,				'''w'''   ,				encoding='''utf-8'''				) as outfile:
                       json.dump(lowercase   ,				lowercase				)

                   # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                   # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                   # "special_tokens_map.json" files
                   UpperCAmelCase			       =		tokenizer_class.from_pretrained(
                       lowercase   ,				)
                   self.assertIn(
                       '''an_additional_special_token'''   ,				tokenizer_without_change_in_init.additional_special_tokens				)
                   self.assertEqual(
                       ['''an_additional_special_token''']   ,				tokenizer_without_change_in_init.convert_ids_to_tokens(
                           tokenizer_without_change_in_init.convert_tokens_to_ids(['''an_additional_special_token''']				)				)   ,				)

                   # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                   UpperCAmelCase			       =		added_tokens_extra_ids + [AddedToken('''a_new_additional_special_token'''   ,				lstrip=lowercase				)]
                   UpperCAmelCase			       =		tokenizer_class.from_pretrained(
                       lowercase   ,				additional_special_tokens=lowercase   ,				)

                   self.assertIn('''a_new_additional_special_token'''   ,				tokenizer.additional_special_tokens				)
                   self.assertEqual(
                       ['''a_new_additional_special_token''']   ,				tokenizer.convert_ids_to_tokens(
                           tokenizer.convert_tokens_to_ids(['''a_new_additional_special_token''']				)				)   ,				)
       def     A      (	self						:	Optional[int]				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.perceiver_tokenizer
           self.assertEqual(tokenizer.decode([178]				)   ,				'''�'''				)
       def     A      (	self						:	Union[str, Any]				):



           '''simple docstring'''

           pass
       def     A      (	self						:	Any				):



           '''simple docstring'''

           pass
       def     A      (	self						:	Dict				):



           '''simple docstring'''

           pass
       def     A      (	self						:	str				):



           '''simple docstring'''

           pass







       def     A      (	self						:	List[str]				):



           '''simple docstring'''

           UpperCAmelCase			       =		self.get_tokenizers(fast=lowercase   ,				do_lower_case=lowercase				)
           for tokenizer in tokenizers:
               with self.subTest(f"{tokenizer.__class__.__name__}"				):
                   UpperCAmelCase			       =		['''[CLS]''', '''t''', '''h''', '''i''', '''s''', ''' ''', '''i''', '''s''', ''' ''', '''a''', ''' ''', '''t''', '''e''', '''s''', '''t''', '''[SEP]''']
                   UpperCAmelCase			       =		tokenizer.convert_tokens_to_string(lowercase				)

                   self.assertIsInstance(lowercase   ,				lowercase				)




