






'''simple docstring'''







import importlib
import json
import os
from collections import OrderedDict
from typing import Dict, Optional, Union

# Build the list of all feature extractors
from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...feature_extraction_utils import FeatureExtractionMixin
from ...utils import CONFIG_NAME, FEATURE_EXTRACTOR_NAME, get_file_from_repo, logging
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)


_A	:			Any         =logging.get_logger(__name__)

_A	:			List[str]         =OrderedDict(
    [
        ('''audio-spectrogram-transformer''', '''ASTFeatureExtractor'''),
        ('''beit''', '''BeitFeatureExtractor'''),
        ('''chinese_clip''', '''ChineseCLIPFeatureExtractor'''),
        ('''clap''', '''ClapFeatureExtractor'''),
        ('''clip''', '''CLIPFeatureExtractor'''),
        ('''clipseg''', '''ViTFeatureExtractor'''),
        ('''conditional_detr''', '''ConditionalDetrFeatureExtractor'''),
        ('''convnext''', '''ConvNextFeatureExtractor'''),
        ('''cvt''', '''ConvNextFeatureExtractor'''),
        ('''data2vec-audio''', '''Wav2Vec2FeatureExtractor'''),
        ('''data2vec-vision''', '''BeitFeatureExtractor'''),
        ('''deformable_detr''', '''DeformableDetrFeatureExtractor'''),
        ('''deit''', '''DeiTFeatureExtractor'''),
        ('''detr''', '''DetrFeatureExtractor'''),
        ('''dinat''', '''ViTFeatureExtractor'''),
        ('''donut-swin''', '''DonutFeatureExtractor'''),
        ('''dpt''', '''DPTFeatureExtractor'''),
        ('''encodec''', '''EncodecFeatureExtractor'''),
        ('''flava''', '''FlavaFeatureExtractor'''),
        ('''glpn''', '''GLPNFeatureExtractor'''),
        ('''groupvit''', '''CLIPFeatureExtractor'''),
        ('''hubert''', '''Wav2Vec2FeatureExtractor'''),
        ('''imagegpt''', '''ImageGPTFeatureExtractor'''),
        ('''layoutlmv2''', '''LayoutLMv2FeatureExtractor'''),
        ('''layoutlmv3''', '''LayoutLMv3FeatureExtractor'''),
        ('''levit''', '''LevitFeatureExtractor'''),
        ('''maskformer''', '''MaskFormerFeatureExtractor'''),
        ('''mctct''', '''MCTCTFeatureExtractor'''),
        ('''mobilenet_v1''', '''MobileNetV1FeatureExtractor'''),
        ('''mobilenet_v2''', '''MobileNetV2FeatureExtractor'''),
        ('''mobilevit''', '''MobileViTFeatureExtractor'''),
        ('''nat''', '''ViTFeatureExtractor'''),
        ('''owlvit''', '''OwlViTFeatureExtractor'''),
        ('''perceiver''', '''PerceiverFeatureExtractor'''),
        ('''poolformer''', '''PoolFormerFeatureExtractor'''),
        ('''regnet''', '''ConvNextFeatureExtractor'''),
        ('''resnet''', '''ConvNextFeatureExtractor'''),
        ('''segformer''', '''SegformerFeatureExtractor'''),
        ('''sew''', '''Wav2Vec2FeatureExtractor'''),
        ('''sew-d''', '''Wav2Vec2FeatureExtractor'''),
        ('''speech_to_text''', '''Speech2TextFeatureExtractor'''),
        ('''speecht5''', '''SpeechT5FeatureExtractor'''),
        ('''swiftformer''', '''ViTFeatureExtractor'''),
        ('''swin''', '''ViTFeatureExtractor'''),
        ('''swinv2''', '''ViTFeatureExtractor'''),
        ('''table-transformer''', '''DetrFeatureExtractor'''),
        ('''timesformer''', '''VideoMAEFeatureExtractor'''),
        ('''tvlt''', '''TvltFeatureExtractor'''),
        ('''unispeech''', '''Wav2Vec2FeatureExtractor'''),
        ('''unispeech-sat''', '''Wav2Vec2FeatureExtractor'''),
        ('''van''', '''ConvNextFeatureExtractor'''),
        ('''videomae''', '''VideoMAEFeatureExtractor'''),
        ('''vilt''', '''ViltFeatureExtractor'''),
        ('''vit''', '''ViTFeatureExtractor'''),
        ('''vit_mae''', '''ViTFeatureExtractor'''),
        ('''vit_msn''', '''ViTFeatureExtractor'''),
        ('''wav2vec2''', '''Wav2Vec2FeatureExtractor'''),
        ('''wav2vec2-conformer''', '''Wav2Vec2FeatureExtractor'''),
        ('''wavlm''', '''Wav2Vec2FeatureExtractor'''),
        ('''whisper''', '''WhisperFeatureExtractor'''),
        ('''xclip''', '''CLIPFeatureExtractor'''),
        ('''yolos''', '''YolosFeatureExtractor'''),
    ]
)

_A	:			Dict         =_LazyAutoMapping(CONFIG_MAPPING_NAMES, FEATURE_EXTRACTOR_MAPPING_NAMES)





def  SCREAMING_SNAKE_CASE_       (UpperCamelCase      )      ->       str:
							for module_name, extractors in FEATURE_EXTRACTOR_MAPPING_NAMES.items():
														if class_name in extractors:
																					lowerCamelCase__					:	List[Any]         =       model_type_to_module_name(UpperCamelCase      )

																					lowerCamelCase__					:	Any         =       importlib.import_module(f'''.{module_name}'''				,			"""transformers.models"""      )
																					try:
																												return getattr(UpperCamelCase				,			UpperCamelCase      )
																					except AttributeError:
																												continue

							for _, extractor in FEATURE_EXTRACTOR_MAPPING._extra_content.items():
														if getattr(UpperCamelCase				,			"""__name__"""				,			UpperCamelCase      ) == class_name:
																					return extractor

    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
							lowerCamelCase__					:	List[Any]         =       importlib.import_module("""transformers"""      )
							if hasattr(UpperCamelCase				,			UpperCamelCase      ):
														return getattr(UpperCamelCase				,			UpperCamelCase      )

							return None





def  SCREAMING_SNAKE_CASE_       (UpperCamelCase				,			UpperCamelCase = None				,			UpperCamelCase = False				,			UpperCamelCase = False				,			UpperCamelCase = None				,			UpperCamelCase = None				,			UpperCamelCase = None				,			UpperCamelCase = False				,			**UpperCamelCase				,			)      ->       Optional[Any]:
							lowerCamelCase__					:	Optional[Any]         =       get_file_from_repo(
							    UpperCamelCase				,			UpperCamelCase				,			cache_dir=UpperCamelCase				,			force_download=UpperCamelCase				,			resume_download=UpperCamelCase				,			proxies=UpperCamelCase				,			use_auth_token=UpperCamelCase				,			revision=UpperCamelCase				,			local_files_only=UpperCamelCase				,			)
							if resolved_config_file is None:
														logger.info(
														    """Could not locate the feature extractor configuration file, will try to use the model config instead."""      )
														return {}

							with open(UpperCamelCase				,			encoding="""utf-8"""      ) as reader:
														return json.load(UpperCamelCase      )



class _lowercase	:




							def __init__(							self:				Optional[int]     ):
														raise EnvironmentError(
														    """AutoFeatureExtractor is designed to be instantiated """
														    """using the `AutoFeatureExtractor.from_pretrained(pretrained_model_name_or_path)` method."""     )




							@classmethod
							@replace_list_option_in_docstrings(UpperCamelCase__     )
							def 		lowerCamelCase_				(							cls:				List[Any]			,							UpperCamelCase__:				int			,							**UpperCamelCase__:				str     ):
														lowerCamelCase__					:	List[Any]         =       kwargs.pop("""config"""			,							UpperCamelCase__     )
														lowerCamelCase__					:	int         =       kwargs.pop("""trust_remote_code"""			,							UpperCamelCase__     )
														lowerCamelCase__					:	Dict         =       True

														lowerCamelCase__  ,		lowerCamelCase__					:	Optional[Any]         =       FeatureExtractionMixin.get_feature_extractor_dict(UpperCamelCase__			,							**UpperCamelCase__     )
														lowerCamelCase__					:	List[str]         =       config_dict.get("""feature_extractor_type"""			,							UpperCamelCase__     )
														lowerCamelCase__					:	Tuple         =       None
														if "AutoFeatureExtractor" in config_dict.get("""auto_map"""			,							{}     ):
																					lowerCamelCase__					:	Any         =       config_dict["""auto_map"""]["""AutoFeatureExtractor"""]

														# If we don't find the feature extractor class in the feature extractor config, let's try the model config.
														if feature_extractor_class is None and feature_extractor_auto_map is None:
																					if not isinstance(UpperCamelCase__			,							UpperCamelCase__     ):
																												lowerCamelCase__					:	List[str]         =       AutoConfig.from_pretrained(UpperCamelCase__			,							**UpperCamelCase__     )
																					# It could be in `config.feature_extractor_type``
																					lowerCamelCase__					:	Tuple         =       getattr(UpperCamelCase__			,							"""feature_extractor_type"""			,							UpperCamelCase__     )
																					if hasattr(UpperCamelCase__			,							"""auto_map"""     ) and "AutoFeatureExtractor" in config.auto_map:
																												lowerCamelCase__					:	Union[str, Any]         =       config.auto_map["""AutoFeatureExtractor"""]

														if feature_extractor_class is not None:
																					lowerCamelCase__					:	List[Any]         =       feature_extractor_class_from_name(UpperCamelCase__     )

														lowerCamelCase__					:	str         =       feature_extractor_auto_map is not None
														lowerCamelCase__					:	Optional[Any]         =       feature_extractor_class is not None or type(UpperCamelCase__     ) in FEATURE_EXTRACTOR_MAPPING
														lowerCamelCase__					:	Optional[Any]         =       resolve_trust_remote_code(
														    UpperCamelCase__			,							UpperCamelCase__			,							UpperCamelCase__			,							UpperCamelCase__     )

														if has_remote_code and trust_remote_code:
																					lowerCamelCase__					:	str         =       get_class_from_dynamic_module(
																					    UpperCamelCase__			,							UpperCamelCase__			,							**UpperCamelCase__     )
																					lowerCamelCase__					:	str         =       kwargs.pop("""code_revision"""			,							UpperCamelCase__     )
																					if os.path.isdir(UpperCamelCase__     ):
																												feature_extractor_class.register_for_auto_class()
																					return feature_extractor_class.from_dict(UpperCamelCase__			,							**UpperCamelCase__     )
														elif feature_extractor_class is not None:
																					return feature_extractor_class.from_dict(UpperCamelCase__			,							**UpperCamelCase__     )
														# Last try: we use the FEATURE_EXTRACTOR_MAPPING.
														elif type(UpperCamelCase__     ) in FEATURE_EXTRACTOR_MAPPING:
																					lowerCamelCase__					:	int         =       FEATURE_EXTRACTOR_MAPPING[type(UpperCamelCase__     )]
																					return feature_extractor_class.from_dict(UpperCamelCase__			,							**UpperCamelCase__     )

														raise ValueError(
														    F'''Unrecognized feature extractor in {pretrained_model_name_or_path}. Should have a '''
														    F'''`feature_extractor_type` key in its {FEATURE_EXTRACTOR_NAME} of {CONFIG_NAME}, or one of the following '''
														    F'''`model_type` keys in its {CONFIG_NAME}: {', '.join(c for c in FEATURE_EXTRACTOR_MAPPING_NAMES.keys()     )}'''     )




							@staticmethod
							def 		lowerCamelCase_				(							UpperCamelCase__:				Optional[Any]			,							UpperCamelCase__:				Optional[Any]     ):
														FEATURE_EXTRACTOR_MAPPING.register(UpperCamelCase__			,							UpperCamelCase__     )









'''simple docstring'''







import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
			_A	:			List[Any]         ='''pt'''
elif is_tf_available():
			_A	:			Any         ='''tf'''
else:
			_A	:			List[str]         ='''jax'''

class _lowercase	(				_lowercase				,      unittest.TestCase		):
							a				      =       ByTaTokenizer
							a				      =       False




							def 		lowerCamelCase_				(							self:				str     ):
														super().setUp()
														lowerCamelCase__					:	str         =       ByTaTokenizer()
														tokenizer.save_pretrained(self.tmpdirname     )




							@cached_property
							def 		lowerCamelCase_				(							self:				Optional[int]     ):
														return ByTaTokenizer.from_pretrained("""google/byt5-small"""     )




							def 		lowerCamelCase_				(							self:				Any			,							**UpperCamelCase__:				Tuple     ):
														return self.tokenizer_class.from_pretrained(self.tmpdirname			,							**UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				Union[str, Any]			,							UpperCamelCase__:				str			,							UpperCamelCase__:				Any=False			,							UpperCamelCase__:				Union[str, Any]=20			,							UpperCamelCase__:				Optional[int]=5     ):
														# XXX The default common tokenizer tests assume that every ID is decodable on its own.
														# This assumption is invalid for ByT5 because single bytes might not be
														# valid utf-8 (byte 128 for instance).
														# Here we're overriding the smallest possible method to provide
														# a clean sequence without making the same assumption.

														lowerCamelCase__					:	List[str]         =       []
														for i in range(len(UpperCamelCase__     )     ):
																					try:
																												lowerCamelCase__					:	Tuple         =       tokenizer.decode([i]			,							clean_up_tokenization_spaces=UpperCamelCase__     )
																					except UnicodeDecodeError:
																												pass
																					toks.append((i, tok)     )

														lowerCamelCase__					:	Union[str, Any]         =       list(filter(lambda UpperCamelCase__     : re.match(R"""^[ a-zA-Z]+$"""			,							t[1]     )			,							UpperCamelCase__     )     )
														lowerCamelCase__					:	Tuple         =       list(filter(lambda UpperCamelCase__     : [t[0]] == tokenizer.encode(t[1]			,							add_special_tokens=UpperCamelCase__     )			,							UpperCamelCase__     )     )
														if max_length is not None and len(UpperCamelCase__     ) > max_length:
																					lowerCamelCase__					:	Dict         =       toks[:max_length]
														if min_length is not None and len(UpperCamelCase__     ) < min_length and len(UpperCamelCase__     ) > 0:
																					while len(UpperCamelCase__     ) < min_length:
																												lowerCamelCase__					:	Optional[Any]         =       toks + toks
        # toks_str = [t[1] for t in toks]
														lowerCamelCase__					:	Union[str, Any]         =       [t[0] for t in toks]

														# Ensure consistency
														lowerCamelCase__					:	Union[str, Any]         =       tokenizer.decode(UpperCamelCase__			,							clean_up_tokenization_spaces=UpperCamelCase__     )
														if " " not in output_txt and len(UpperCamelCase__     ) > 1:
																					lowerCamelCase__					:	Optional[int]         =       (
																					    tokenizer.decode([toks_ids[0]]			,							clean_up_tokenization_spaces=UpperCamelCase__     )
																					    + """ """
																					    + tokenizer.decode(toks_ids[1:]			,							clean_up_tokenization_spaces=UpperCamelCase__     )
																					)
														if with_prefix_space:
																					lowerCamelCase__					:	str         =       """ """ + output_txt
														lowerCamelCase__					:	List[Any]         =       tokenizer.encode(UpperCamelCase__			,							add_special_tokens=UpperCamelCase__     )
														return output_txt, output_ids




							def 		lowerCamelCase_				(							self:				Tuple     ):
														lowerCamelCase__					:	str         =       self.ta_base_tokenizer
														lowerCamelCase__					:	Union[str, Any]         =       tokenizer(["""hi</s>""", """I went to the gym</s>""", """</s>"""]     )
														lowerCamelCase__					:	Optional[int]         =       tokenizer(["""hi""", """I went to the gym""", """"""]     )
														self.assertListEqual(batch_with_eos_added["""input_ids"""]			,							batch_without_eos_added["""input_ids"""]     )




							def 		lowerCamelCase_				(							self:				Tuple     ):
														lowerCamelCase__					:	Optional[Any]         =       self.ta_base_tokenizer
														lowerCamelCase__					:	Dict         =       """Unicode €."""
														lowerCamelCase__					:	List[Any]         =       tokenizer(UpperCamelCase__     )
														lowerCamelCase__					:	List[str]         =       [88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
														self.assertEqual(encoded["""input_ids"""]			,							UpperCamelCase__     )

														# decoding
														lowerCamelCase__					:	Tuple         =       tokenizer.decode(UpperCamelCase__     )
														self.assertEqual(UpperCamelCase__			,							"""Unicode €.</s>"""     )

														lowerCamelCase__					:	List[Any]         =       tokenizer("""e è é ê ë"""     )
														lowerCamelCase__					:	List[Any]         =       [104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
														self.assertEqual(encoded["""input_ids"""]			,							UpperCamelCase__     )
														# decoding
														lowerCamelCase__					:	str         =       tokenizer.decode(UpperCamelCase__     )
														self.assertEqual(UpperCamelCase__			,							"""e è é ê ë</s>"""     )

														# encode/decode, but with `encode` instead of `__call__`
														self.assertEqual(tokenizer.decode(tokenizer.encode("""e è é ê ë"""     )     )			,							"""e è é ê ë</s>"""     )




							def 		lowerCamelCase_				(							self:				Any     ):
														lowerCamelCase__					:	int         =       self.ta_base_tokenizer
														lowerCamelCase__					:	Tuple         =       ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
														# fmt: off
														lowerCamelCase__					:	List[str]         =       [68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
														# fmt: on
														lowerCamelCase__					:	int         =       tokenizer(UpperCamelCase__			,							padding=UpperCamelCase__			,							return_tensors=UpperCamelCase__     )
														self.assertIsInstance(UpperCamelCase__			,							UpperCamelCase__     )

														if FRAMEWORK != "jax":
																					lowerCamelCase__					:	Any         =       list(batch.input_ids.numpy()[0]     )
														else:
																					lowerCamelCase__					:	str         =       list(batch.input_ids.tolist()[0]     )

														self.assertListEqual(UpperCamelCase__			,							UpperCamelCase__     )

														self.assertEqual((2, 37)			,							batch.input_ids.shape     )
														self.assertEqual((2, 37)			,							batch.attention_mask.shape     )




							def 		lowerCamelCase_				(							self:				List[Any]     ):
														lowerCamelCase__					:	List[str]         =       self.ta_base_tokenizer
														lowerCamelCase__					:	Tuple         =       ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
														lowerCamelCase__					:	Tuple         =       tokenizer(UpperCamelCase__			,							padding=UpperCamelCase__			,							return_tensors=UpperCamelCase__     )
														# check if input_ids are returned and no decoder_input_ids
														self.assertIn("""input_ids"""			,							UpperCamelCase__     )
														self.assertIn("""attention_mask"""			,							UpperCamelCase__     )
														self.assertNotIn("""decoder_input_ids"""			,							UpperCamelCase__     )
														self.assertNotIn("""decoder_attention_mask"""			,							UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				Tuple     ):
														lowerCamelCase__					:	str         =       self.ta_base_tokenizer
														lowerCamelCase__					:	List[Any]         =       [
														    """Summary of the text.""",
														    """Another summary.""",
														]
														lowerCamelCase__					:	Union[str, Any]         =       tokenizer(
														    text_target=UpperCamelCase__			,							max_length=32			,							padding="""max_length"""			,							truncation=UpperCamelCase__			,							return_tensors=UpperCamelCase__     )
														self.assertEqual(32			,							targets["""input_ids"""].shape[1]     )




							def 		lowerCamelCase_				(							self:				str     ):
														lowerCamelCase__					:	Tuple         =       self.ta_base_tokenizer
														lowerCamelCase__					:	str         =       ["""A long paragraph for summarization. </s>"""]
														lowerCamelCase__					:	Optional[Any]         =       ["""Summary of the text. </s>"""]
														# fmt: off
														lowerCamelCase__					:	Tuple         =       [68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
														lowerCamelCase__					:	Any         =       [86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
														# fmt: on

														lowerCamelCase__					:	Any         =       tokenizer(UpperCamelCase__			,							text_target=UpperCamelCase__     )

														self.assertEqual(UpperCamelCase__			,							batch["""input_ids"""][0]     )
														self.assertEqual(UpperCamelCase__			,							batch["""labels"""][0]     )




							def 		lowerCamelCase_				(							self:				Optional[int]     ):
														# safety check on max_len default value so we are sure the test works
														lowerCamelCase__					:	Any         =       self.get_tokenizers()
														for tokenizer in tokenizers:
																					with self.subTest(F'''{tokenizer.__class__.__name__}'''     ):
																												self.assertNotEqual(tokenizer.model_max_length			,							42     )

        # Now let's start the test
														lowerCamelCase__					:	Any         =       self.get_tokenizers()
														for tokenizer in tokenizers:
																					with self.subTest(F'''{tokenizer.__class__.__name__}'''     ):
																												# Isolate this from the other tests because we save additional tokens/etc
																												lowerCamelCase__					:	int         =       tempfile.mkdtemp()

																												lowerCamelCase__					:	List[str]         =       """ He is very happy, UNwant\u00E9d,running"""
																												lowerCamelCase__					:	List[Any]         =       tokenizer.encode(UpperCamelCase__			,							add_special_tokens=UpperCamelCase__     )
																												tokenizer.save_pretrained(UpperCamelCase__     )

																												lowerCamelCase__					:	List[Any]         =       tokenizer.__class__.from_pretrained(UpperCamelCase__     )
																												lowerCamelCase__					:	Optional[int]         =       after_tokenizer.encode(UpperCamelCase__			,							add_special_tokens=UpperCamelCase__     )
																												self.assertListEqual(UpperCamelCase__			,							UpperCamelCase__     )

																												shutil.rmtree(UpperCamelCase__     )

														lowerCamelCase__					:	Any         =       self.get_tokenizers(model_max_length=42     )
														for tokenizer in tokenizers:
																					with self.subTest(F'''{tokenizer.__class__.__name__}'''     ):
																												# Isolate this from the other tests because we save additional tokens/etc
																												lowerCamelCase__					:	Any         =       tempfile.mkdtemp()

																												lowerCamelCase__					:	Optional[Any]         =       """ He is very happy, UNwant\u00E9d,running"""
																												tokenizer.add_tokens(["""bim""", """bambam"""]     )
																												lowerCamelCase__					:	Optional[int]         =       tokenizer.additional_special_tokens
																												additional_special_tokens.append("""new_additional_special_token"""     )
																												tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}     )
																												lowerCamelCase__					:	List[Any]         =       tokenizer.encode(UpperCamelCase__			,							add_special_tokens=UpperCamelCase__     )
																												tokenizer.save_pretrained(UpperCamelCase__     )

																												lowerCamelCase__					:	Union[str, Any]         =       tokenizer.__class__.from_pretrained(UpperCamelCase__     )
																												lowerCamelCase__					:	int         =       after_tokenizer.encode(UpperCamelCase__			,							add_special_tokens=UpperCamelCase__     )
																												self.assertListEqual(UpperCamelCase__			,							UpperCamelCase__     )
																												self.assertIn("""new_additional_special_token"""			,							after_tokenizer.additional_special_tokens     )
																												self.assertEqual(after_tokenizer.model_max_length			,							42     )

																												lowerCamelCase__					:	Any         =       tokenizer.__class__.from_pretrained(UpperCamelCase__			,							model_max_length=43     )
																												self.assertEqual(tokenizer.model_max_length			,							43     )

																												shutil.rmtree(UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				Optional[int]     ):
														lowerCamelCase__					:	List[Any]         =       []
														if self.test_slow_tokenizer:
																					tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())     )

														if self.test_rust_tokenizer:
																					tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())     )

														for tokenizer_class, tokenizer_utils in tokenizer_list:
																					with tempfile.TemporaryDirectory() as tmp_dir:
																												tokenizer_utils.save_pretrained(UpperCamelCase__     )

																												with open(os.path.join(UpperCamelCase__			,							"""special_tokens_map.json"""     )			,							encoding="""utf-8"""     ) as json_file:
																																			lowerCamelCase__					:	Union[str, Any]         =       json.load(UpperCamelCase__     )

																												with open(os.path.join(UpperCamelCase__			,							"""tokenizer_config.json"""     )			,							encoding="""utf-8"""     ) as json_file:
																																			lowerCamelCase__					:	Optional[Any]         =       json.load(UpperCamelCase__     )

																												lowerCamelCase__					:	Optional[int]         =       [F'''<extra_id_{i}>''' for i in range(125     )]

																												lowerCamelCase__					:	int         =       added_tokens_extra_ids + [
																												    """an_additional_special_token"""
																												]
																												lowerCamelCase__					:	Optional[Any]         =       added_tokens_extra_ids + [
																												    """an_additional_special_token"""
																												]

																												with open(os.path.join(UpperCamelCase__			,							"""special_tokens_map.json"""     )			,							"""w"""			,							encoding="""utf-8"""     ) as outfile:
																																			json.dump(UpperCamelCase__			,							UpperCamelCase__     )
																												with open(os.path.join(UpperCamelCase__			,							"""tokenizer_config.json"""     )			,							"""w"""			,							encoding="""utf-8"""     ) as outfile:
																																			json.dump(UpperCamelCase__			,							UpperCamelCase__     )

																												# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																												# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																												# "special_tokens_map.json" files
																												lowerCamelCase__					:	Dict         =       tokenizer_class.from_pretrained(
																												    UpperCamelCase__			,							)
																												self.assertIn(
																												    """an_additional_special_token"""			,							tokenizer_without_change_in_init.additional_special_tokens     )
																												# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
																												self.assertEqual(
																												    ["""an_additional_special_token"""]			,							tokenizer_without_change_in_init.convert_ids_to_tokens(
																												        tokenizer_without_change_in_init.convert_tokens_to_ids(["""an_additional_special_token"""]     )     )			,							)

																												# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																												lowerCamelCase__					:	Optional[Any]         =       added_tokens_extra_ids + [AddedToken("""a_new_additional_special_token"""			,							lstrip=UpperCamelCase__     )]
																												lowerCamelCase__					:	Any         =       tokenizer_class.from_pretrained(
																												    UpperCamelCase__			,							additional_special_tokens=UpperCamelCase__			,							)

																												self.assertIn("""a_new_additional_special_token"""			,							tokenizer.additional_special_tokens     )
																												self.assertEqual(
																												    ["""a_new_additional_special_token"""]			,							tokenizer.convert_ids_to_tokens(
																												        tokenizer.convert_tokens_to_ids(["""a_new_additional_special_token"""]     )     )			,							)




							def 		lowerCamelCase_				(							self:				Union[str, Any]     ):
														lowerCamelCase__					:	Dict         =       []
														if self.test_slow_tokenizer:
																					tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())     )

														if self.test_rust_tokenizer:
																					tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())     )

														for tokenizer_class, tokenizer_utils in tokenizer_list:
																					with tempfile.TemporaryDirectory() as tmp_dir:
																												tokenizer_utils.save_pretrained(UpperCamelCase__     )

																												lowerCamelCase__					:	str         =       tokenizer_class.from_pretrained(UpperCamelCase__     )

																												self.assertTrue(tokenizer.decode([255]     ) == """"""     )




							def 		lowerCamelCase_				(							self:				Optional[int]     ):
														pass




							def 		lowerCamelCase_				(							self:				str     ):
														pass




							def 		lowerCamelCase_				(							self:				List[str]     ):
														pass




							def 		lowerCamelCase_				(							self:				Optional[int]     ):
														pass




							def 		lowerCamelCase_				(							self:				int     ):
														# The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
														# and special added tokens as tokens
														lowerCamelCase__					:	Dict         =       self.get_tokenizers(fast=UpperCamelCase__			,							do_lower_case=UpperCamelCase__     )
														for tokenizer in tokenizers:
																					with self.subTest(F'''{tokenizer.__class__.__name__}'''     ):
																												lowerCamelCase__					:	Dict         =       ["""t""", """h""", """i""", """s""", """ """, """i""", """s""", """ """, """a""", """ """, """t""", """e""", """x""", """t""", """</s>"""]
																												lowerCamelCase__					:	Optional[int]         =       tokenizer.convert_tokens_to_string(UpperCamelCase__     )

																												self.assertIsInstance(UpperCamelCase__			,							UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				Any     ):
														lowerCamelCase__					:	str         =       self.get_tokenizers()
														for tokenizer in tokenizers:
																					with self.subTest(F'''{tokenizer.__class__.__name__}'''     ):
																												lowerCamelCase__					:	str         =       [
																												    """bos_token""",
																												    """eos_token""",
																												    """unk_token""",
																												    """sep_token""",
																												    """pad_token""",
																												    """cls_token""",
																												    """mask_token""",
																												]

																												lowerCamelCase__					:	str         =       0
																												lowerCamelCase__					:	Any         =       tokenizer.convert_ids_to_tokens(
																												    UpperCamelCase__			,							skip_special_tokens=UpperCamelCase__     )

																												for attr in attributes_list:
																																			setattr(UpperCamelCase__			,							attr + """_id"""			,							UpperCamelCase__     )
																																			self.assertEqual(getattr(UpperCamelCase__			,							UpperCamelCase__     )			,							UpperCamelCase__     )
																																			self.assertEqual(getattr(UpperCamelCase__			,							attr + """_id"""     )			,							UpperCamelCase__     )

																																			setattr(UpperCamelCase__			,							attr + """_id"""			,							UpperCamelCase__     )
																																			self.assertEqual(getattr(UpperCamelCase__			,							UpperCamelCase__     )			,							UpperCamelCase__     )
																																			self.assertEqual(getattr(UpperCamelCase__			,							attr + """_id"""     )			,							UpperCamelCase__     )

																												setattr(UpperCamelCase__			,							"""additional_special_tokens_ids"""			,							[]     )
																												self.assertListEqual(getattr(UpperCamelCase__			,							"""additional_special_tokens"""     )			,							[]     )
																												self.assertListEqual(getattr(UpperCamelCase__			,							"""additional_special_tokens_ids"""     )			,							[]     )

																												setattr(UpperCamelCase__			,							"""additional_special_tokens_ids"""			,							[token_id_to_test_setters]     )
																												self.assertListEqual(getattr(UpperCamelCase__			,							"""additional_special_tokens"""     )			,							[token_to_test_setters]     )
																												self.assertListEqual(getattr(UpperCamelCase__			,							"""additional_special_tokens_ids"""     )			,							[token_id_to_test_setters]     )

