





'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


lowerCAmelCase_					:    Any      	=			logging.get_logger(__name__)

lowerCAmelCase_					:    int      	=			{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowerCAmelCase_					:    List[str]      	=			[
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

lowerCAmelCase_					:    str      	=			{
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
lowerCAmelCase_					:    List[Any]      	=			{f"""funnel-transformer/{name}""": 5_12 for name in _model_names}
lowerCAmelCase_					:    List[Any]      	=			{f"""funnel-transformer/{name}""": {'do_lower_case': True} for name in _model_names}
class 				__SCREAMING_SNAKE_CASE  (lowerCamelCase_      ):






				"""simple docstring"""







				__a            =VOCAB_FILES_NAMES
				__a            =PRETRAINED_VOCAB_FILES_MAP
				__a            =PRETRAINED_INIT_CONFIGURATION
				__a            =FunnelTokenizer
				__a            =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__a            =2




				def __init__(  self     :					str					,				__a     :					Optional[Any]=None					,				__a     :					Optional[Any]=None					,				__a     :					List[str]=True					,				__a     :					str="<unk>"					,				__a     :					List[Any]="<sep>"					,				__a     :					Optional[int]="<pad>"					,				__a     :					str="<cls>"					,				__a     :					Union[str, Any]="<mask>"					,				__a     :					Dict="<s>"					,				__a     :					Tuple="</s>"					,				__a     :					Union[str, Any]=True					,				__a     :					str=True					,				__a     :					int=None					,				__a     :					int="##"					,				**__a     :					List[Any]					,				):
								super().__init__(
								    __a					,				tokenizer_file=__a					,				do_lower_case=__a					,				unk_token=__a					,				sep_token=__a					,				pad_token=__a					,				cls_token=__a					,				mask_token=__a					,				bos_token=__a					,				eos_token=__a					,				clean_text=__a					,				tokenize_chinese_chars=__a					,				strip_accents=__a					,				wordpieces_prefix=__a					,				**__a					,				)

								_a								= json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
								if (
								    normalizer_state.get("lowercase"					,				__a					) != do_lower_case
								    or normalizer_state.get("strip_accents"					,				__a					) != strip_accents
								    or normalizer_state.get("handle_chinese_chars"					,				__a					) != tokenize_chinese_chars
								):
												_a								= getattr(__a					,				normalizer_state.pop("type"					)					)
												_a								= do_lower_case
												_a								= strip_accents
												_a								= tokenize_chinese_chars
												_a								= normalizer_class(**__a					)

								_a								= do_lower_case




				def   UpperCamelCase__   (  self     :					Any					,				__a     :					Any					,				__a     :					Dict=None					):
								_a								= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
												output += token_ids_a + [self.sep_token_id]

								return output




				def   UpperCamelCase__   (  self     :					str					,				__a     :					List[int]					,				__a     :					Optional[List[int]] = None					):
								_a								= [self.sep_token_id]
								_a								= [self.cls_token_id]
								if token_ids_a is None:
												return len(cls					) * [self.cls_token_type_id] + len(token_ids_a + sep					) * [0]
								return len(cls					) * [self.cls_token_type_id] + len(token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]




				def   UpperCamelCase__   (  self     :					str					,				__a     :					str					,				__a     :					Optional[str] = None					):
								_a								= self._tokenizer.model.save(__a					,				name=__a					)
								return tuple(__a					)







'''simple docstring'''







from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowerCAmelCase_					:    int      	=			logging.get_logger(__name__)

lowerCAmelCase_					:    Tuple      	=			{
    'google/bigbird-roberta-base': 'https://huggingface.co/google/bigbird-roberta-base/resolve/main/config.json',
    'google/bigbird-roberta-large': 'https://huggingface.co/google/bigbird-roberta-large/resolve/main/config.json',
    'google/bigbird-base-trivia-itc': 'https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/config.json',
    # See all BigBird models at https://huggingface.co/models?filter=big_bird
}
class 				__SCREAMING_SNAKE_CASE  (lowerCamelCase_      ):






				"""simple docstring"""







				__a            ='big_bird'




				def __init__(  self     :					Optional[int]					,				__a     :					Dict=5_03_58					,				__a     :					str=7_68					,				__a     :					List[Any]=12					,				__a     :					List[str]=12					,				__a     :					Union[str, Any]=30_72					,				__a     :					str="gelu_new"					,				__a     :					Dict=0.1					,				__a     :					Union[str, Any]=0.1					,				__a     :					Any=40_96					,				__a     :					int=2					,				__a     :					Tuple=0.02					,				__a     :					List[Any]=1e-1_2					,				__a     :					int=True					,				__a     :					List[str]=0					,				__a     :					Tuple=1					,				__a     :					Optional[Any]=2					,				__a     :					Tuple=66					,				__a     :					str="block_sparse"					,				__a     :					Tuple=True					,				__a     :					Optional[int]=False					,				__a     :					str=64					,				__a     :					Tuple=3					,				__a     :					Any=None					,				**__a     :					Dict					,				):
								super().__init__(
								    pad_token_id=__a					,				bos_token_id=__a					,				eos_token_id=__a					,				sep_token_id=__a					,				**__a					,				)

								_a								= vocab_size
								_a								= max_position_embeddings
								_a								= hidden_size
								_a								= num_hidden_layers
								_a								= num_attention_heads
								_a								= intermediate_size
								_a								= hidden_act
								_a								= hidden_dropout_prob
								_a								= attention_probs_dropout_prob
								_a								= initializer_range
								_a								= type_vocab_size
								_a								= layer_norm_eps
								_a								= use_cache

								_a								= rescale_embeddings
								_a								= attention_type
								_a								= use_bias
								_a								= block_size
								_a								= num_random_blocks
								_a								= classifier_dropout


class 				__SCREAMING_SNAKE_CASE  (lowerCamelCase_      ):






				"""simple docstring"""







				@property
				def   UpperCamelCase__   (  self     :					Optional[int]					):
								if self.task == "multiple-choice":
												_a								= {0: "batch", 1: "choice", 2: "sequence"}
								else:
												_a								= {0: "batch", 1: "sequence"}
								return OrderedDict(
								    [
								        ("input_ids", dynamic_axis),
								        ("attention_mask", dynamic_axis),
								    ]					)
