


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__lowerCAmelCase	: Any  				=logging.get_logger(__name__)

__lowerCAmelCase	: int  				={'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__lowerCAmelCase	: List[str]  				=[
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

__lowerCAmelCase	: str  				={
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
__lowerCAmelCase	: List[Any]  				={f"""funnel-transformer/{name}""": 5_1_2 for name in _model_names}
__lowerCAmelCase	: List[Any]  				={f"""funnel-transformer/{name}""": {'do_lower_case': True} for name in _model_names}
class        _lowercase      (			lowerCamelCase_       ):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE__				:			List[str]						=		VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE__				:			List[str]						=		PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE__				:			Dict						=		PRETRAINED_INIT_CONFIGURATION
				SCREAMING_SNAKE_CASE__				:			Dict						=		FunnelTokenizer
				SCREAMING_SNAKE_CASE__				:			Optional[Any]						=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE__				:			str						=		2


				def __init__(       self			:str			,			lowerCAmelCase__			:Optional[Any]=None			,			lowerCAmelCase__			:Optional[Any]=None			,			lowerCAmelCase__			:List[str]=True			,			lowerCAmelCase__			:str="<unk>"			,			lowerCAmelCase__			:List[Any]="<sep>"			,			lowerCAmelCase__			:Optional[int]="<pad>"			,			lowerCAmelCase__			:str="<cls>"			,			lowerCAmelCase__			:Union[str, Any]="<mask>"			,			lowerCAmelCase__			:Dict="<s>"			,			lowerCAmelCase__			:Tuple="</s>"			,			lowerCAmelCase__			:Union[str, Any]=True			,			lowerCAmelCase__			:str=True			,			lowerCAmelCase__			:int=None			,			lowerCAmelCase__			:int="##"			,			**lowerCAmelCase__			:List[Any]			,			)		->			Optional[Any]:
										super().__init__(
										    __a			,			tokenizer_file=__a			,			do_lower_case=__a			,			unk_token=__a			,			sep_token=__a			,			pad_token=__a			,			cls_token=__a			,			mask_token=__a			,			bos_token=__a			,			eos_token=__a			,			clean_text=__a			,			tokenize_chinese_chars=__a			,			strip_accents=__a			,			wordpieces_prefix=__a			,			**__a			,			)

										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
										if (
										    normalizer_state.get('''lowercase'''			,			__a	) != do_lower_case
										    or normalizer_state.get('''strip_accents'''			,			__a	) != strip_accents
										    or normalizer_state.get('''handle_chinese_chars'''			,			__a	) != tokenize_chinese_chars
										):
																__SCREAMING_SNAKE_CASE						:     Any					=		getattr(__a			,			normalizer_state.pop('''type'''	)	)
																__SCREAMING_SNAKE_CASE						:     List[Any]					=		do_lower_case
																__SCREAMING_SNAKE_CASE						:     str					=		strip_accents
																__SCREAMING_SNAKE_CASE						:     List[Any]					=		tokenize_chinese_chars
																__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		normalizer_class(**__a	)

										__SCREAMING_SNAKE_CASE						:     Any					=		do_lower_case


				def 		__magic_name__(       self			:Any			,			lowerCAmelCase__			:Any			,			lowerCAmelCase__			:Dict=None	)		->			Dict:
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

										if token_ids_a:
																output += token_ids_a + [self.sep_token_id]

										return output


				def 		__magic_name__(       self			:str			,			lowerCAmelCase__			:List[int]			,			lowerCAmelCase__			:Optional[List[int]] = None	)		->			Tuple:
										__SCREAMING_SNAKE_CASE						:     str					=		[self.sep_token_id]
										__SCREAMING_SNAKE_CASE						:     List[str]					=		[self.cls_token_id]
										if token_ids_a is None:
																return len(cls	) * [self.cls_token_type_id] + len(token_ids_a + sep	) * [0]
										return len(cls	) * [self.cls_token_type_id] + len(token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]



				def 		__magic_name__(       self			:str			,			lowerCAmelCase__			:str			,			lowerCAmelCase__			:Optional[str] = None	)		->			Optional[Any]:
										__SCREAMING_SNAKE_CASE						:     int					=		self._tokenizer.model.save(__a			,			name=__a	)
										return tuple(__a	)







'''simple docstring'''







import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402


lowerCAmelCase_					:    Any      	=			get_tests_dir('fixtures')
lowerCAmelCase_					:    Union[str, Any]      	=			get_tests_dir('fixtures/dummy_feature_extractor_config.json')
lowerCAmelCase_					:    Dict      	=			get_tests_dir('fixtures/dummy-config.json')
class 				__SCREAMING_SNAKE_CASE  (unittest.TestCase      ):






				"""simple docstring"""







				def   UpperCamelCase__   (  self     :					Optional[int]					):
								_a								= 0




				def   UpperCamelCase__   (  self     :					str					):
								_a								= AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-base-960h"					)
								self.assertIsInstance(__a					,				__a					)




				def   UpperCamelCase__   (  self     :					Tuple					):
								_a								= AutoFeatureExtractor.from_pretrained(__a					)
								self.assertIsInstance(__a					,				__a					)




				def   UpperCamelCase__   (  self     :					List[Any]					):
								with tempfile.TemporaryDirectory() as tmpdirname:
												_a								= WavaVecaConfig()

												# remove feature_extractor_type to make sure config.json alone is enough to load feature processor locally
												_a								= AutoFeatureExtractor.from_pretrained(__a					).to_dict()

												config_dict.pop("feature_extractor_type"					)
												_a								= WavaVecaFeatureExtractor(**__a					)

												# save in new folder
												model_config.save_pretrained(__a					)
												config.save_pretrained(__a					)

												_a								= AutoFeatureExtractor.from_pretrained(__a					)

												# make sure private variable is not incorrectly saved
												_a								= json.loads(config.to_json_string()					)
												self.assertTrue("_processor_class" not in dict_as_saved					)

								self.assertIsInstance(__a					,				__a					)




				def   UpperCamelCase__   (  self     :					Tuple					):
								_a								= AutoFeatureExtractor.from_pretrained(__a					)
								self.assertIsInstance(__a					,				__a					)




				def   UpperCamelCase__   (  self     :					Union[str, Any]					):
								with self.assertRaisesRegex(
								    __a					,				"bert-base is not a local folder and is not a valid model identifier"					):
												_a								= AutoFeatureExtractor.from_pretrained("bert-base"					)




				def   UpperCamelCase__   (  self     :					Optional[Any]					):
								with self.assertRaisesRegex(
								    __a					,				r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"					):
												_a								= AutoFeatureExtractor.from_pretrained(__a					,				revision="aaaaaa"					)




				def   UpperCamelCase__   (  self     :					List[Any]					):
								with self.assertRaisesRegex(
								    __a					,				"hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json."					,				):
												_a								= AutoFeatureExtractor.from_pretrained("hf-internal-testing/config-no-model"					)




				def   UpperCamelCase__   (  self     :					List[Any]					):
								# If remote code is not set, we will time out when asking whether to load the model.
								with self.assertRaises(__a					):
												_a								= AutoFeatureExtractor.from_pretrained(
												    "hf-internal-testing/test_dynamic_feature_extractor"					)
								# If remote code is disabled, we can't load this config.
								with self.assertRaises(__a					):
												_a								= AutoFeatureExtractor.from_pretrained(
												    "hf-internal-testing/test_dynamic_feature_extractor"					,				trust_remote_code=__a					)

								_a								= AutoFeatureExtractor.from_pretrained(
								    "hf-internal-testing/test_dynamic_feature_extractor"					,				trust_remote_code=__a					)
								self.assertEqual(feature_extractor.__class__.__name__					,				"NewFeatureExtractor"					)

								# Test feature extractor can be reloaded.
								with tempfile.TemporaryDirectory() as tmp_dir:
												feature_extractor.save_pretrained(__a					)
												_a								= AutoFeatureExtractor.from_pretrained(__a					,				trust_remote_code=__a					)
								self.assertEqual(reloaded_feature_extractor.__class__.__name__					,				"NewFeatureExtractor"					)




				def   UpperCamelCase__   (  self     :					Any					):
								try:
												AutoConfig.register("custom"					,				__a					)
												AutoFeatureExtractor.register(__a					,				__a					)
												# Trying to register something existing in the Transformers library will raise an error
												with self.assertRaises(__a					):
																AutoFeatureExtractor.register(__a					,				__a					)

												# Now that the config is registered, it can be used as any other config with the auto-API
												_a								= CustomFeatureExtractor.from_pretrained(__a					)
												with tempfile.TemporaryDirectory() as tmp_dir:
																feature_extractor.save_pretrained(__a					)
																_a								= AutoFeatureExtractor.from_pretrained(__a					)
																self.assertIsInstance(__a					,				__a					)

								finally:
												if "custom" in CONFIG_MAPPING._extra_content:
																del CONFIG_MAPPING._extra_content["custom"]
												if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
																del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]




				def   UpperCamelCase__   (  self     :					Tuple					):
								class 				__SCREAMING_SNAKE_CASE  (lowerCamelCase_      ):






												"""simple docstring"""







												__a            =True

								try:
												AutoConfig.register("custom"					,				__a					)
												AutoFeatureExtractor.register(__a					,				__a					)
												# If remote code is not set, the default is to use local
												_a								= AutoFeatureExtractor.from_pretrained(
												    "hf-internal-testing/test_dynamic_feature_extractor"					)
												self.assertEqual(feature_extractor.__class__.__name__					,				"NewFeatureExtractor"					)
												self.assertTrue(feature_extractor.is_local					)

												# If remote code is disabled, we load the local one.
												_a								= AutoFeatureExtractor.from_pretrained(
												    "hf-internal-testing/test_dynamic_feature_extractor"					,				trust_remote_code=__a					)
												self.assertEqual(feature_extractor.__class__.__name__					,				"NewFeatureExtractor"					)
												self.assertTrue(feature_extractor.is_local					)

												# If remote is enabled, we load from the Hub
												_a								= AutoFeatureExtractor.from_pretrained(
												    "hf-internal-testing/test_dynamic_feature_extractor"					,				trust_remote_code=__a					)
												self.assertEqual(feature_extractor.__class__.__name__					,				"NewFeatureExtractor"					)
												self.assertTrue(not hasattr(__a					,				"is_local"					)					)

								finally:
												if "custom" in CONFIG_MAPPING._extra_content:
																del CONFIG_MAPPING._extra_content["custom"]
												if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
																del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
