



'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


UpperCamelCase__:   Optional[int]								=			logging.get_logger(__name__)

UpperCamelCase__:   str								=			{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCamelCase__:   int								=			[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

UpperCamelCase__:   Dict								=			{
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
UpperCamelCase__:   int								=			{F"funnel-transformer/{name}": 512 for name in _model_names}
UpperCamelCase__:   Tuple								=			{F"funnel-transformer/{name}": {"do_lower_case": True} for name in _model_names}

class 				SCREAMING_SNAKE_CASE(							A__			):







	"""simple docstring"""






	lowerCamelCase__			 =       VOCAB_FILES_NAMES
	lowerCamelCase__			 =       PRETRAINED_VOCAB_FILES_MAP
	lowerCamelCase__			 =       PRETRAINED_INIT_CONFIGURATION
	lowerCamelCase__			 =       FunnelTokenizer
	lowerCamelCase__			 =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowerCamelCase__			 =       2







	def __init__(						self			:     List[str]		,			__snake_case			:     str=None		,			__snake_case			:     Tuple=None		,			__snake_case			:     int=True		,			__snake_case			:     Optional[Any]="<unk>"		,			__snake_case			:     List[Any]="<sep>"		,			__snake_case			:     Dict="<pad>"		,			__snake_case			:     List[Any]="<cls>"		,			__snake_case			:     str="<mask>"		,			__snake_case			:     Optional[Any]="<s>"		,			__snake_case			:     List[str]="</s>"		,			__snake_case			:     int=True		,			__snake_case			:     Any=True		,			__snake_case			:     int=None		,			__snake_case			:     List[Any]="##"		,			**__snake_case			:     Tuple		,			)			->		Tuple:
								super().__init__(
								    _snake_case		,			tokenizer_file=_snake_case		,			do_lower_case=_snake_case		,			unk_token=_snake_case		,			sep_token=_snake_case		,			pad_token=_snake_case		,			cls_token=_snake_case		,			mask_token=_snake_case		,			bos_token=_snake_case		,			eos_token=_snake_case		,			clean_text=_snake_case		,			tokenize_chinese_chars=_snake_case		,			strip_accents=_snake_case		,			wordpieces_prefix=_snake_case		,			**_snake_case		,			)

								UpperCAmelCase :       Dict 							=		json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
								if (
								    normalizer_state.get('''lowercase'''		,			_snake_case		) != do_lower_case
								    or normalizer_state.get('''strip_accents'''		,			_snake_case		) != strip_accents
								    or normalizer_state.get('''handle_chinese_chars'''		,			_snake_case		) != tokenize_chinese_chars
								):
															UpperCAmelCase :       Any 							=		getattr(_snake_case		,			normalizer_state.pop('''type'''		)		)
															UpperCAmelCase :       List[Any] 							=		do_lower_case
															UpperCAmelCase :       int 							=		strip_accents
															UpperCAmelCase :       Optional[int] 							=		tokenize_chinese_chars
															UpperCAmelCase :       Union[str, Any] 							=		normalizer_class(**_snake_case		)

								UpperCAmelCase :       Dict 							=		do_lower_case







	def      A	(						self			:     Tuple		,			__snake_case			:     str		,			__snake_case			:     Optional[Any]=None		)			->		Union[str, Any]:
								UpperCAmelCase :       List[Any] 							=		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
															output += token_ids_a + [self.sep_token_id]

								return output







	def      A	(						self			:     List[Any]		,			__snake_case			:     List[int]		,			__snake_case			:     Optional[List[int]] = None		)			->		str:
								UpperCAmelCase :       Optional[Any] 							=		[self.sep_token_id]
								UpperCAmelCase :       Optional[Any] 							=		[self.cls_token_id]
								if token_ids_a is None:
															return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0]
								return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]







	def      A	(						self			:     Tuple		,			__snake_case			:     str		,			__snake_case			:     Optional[str] = None		)			->		Optional[Any]:
								UpperCAmelCase :       str 							=		self._tokenizer.model.save(_snake_case		,			name=_snake_case		)
								return tuple(_snake_case		)











from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


snake_case_ : Dict    		=       {"configuration_mbart": ["MBART_PRETRAINED_CONFIG_ARCHIVE_MAP", "MBartConfig", "MBartOnnxConfig"]}

try:
    if not is_sentencepiece_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    snake_case_ : Tuple    		=       ["MBartTokenizer"]

try:
    if not is_tokenizers_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    snake_case_ : str    		=       ["MBartTokenizerFast"]

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    snake_case_ : List[Any]    		=       [
        "MBART_PRETRAINED_MODEL_ARCHIVE_LIST",
        "MBartForCausalLM",
        "MBartForConditionalGeneration",
        "MBartForQuestionAnswering",
        "MBartForSequenceClassification",
        "MBartModel",
        "MBartPreTrainedModel",
    ]

try:
    if not is_tf_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    snake_case_ : Any    		=       [
        "TFMBartForConditionalGeneration",
        "TFMBartModel",
        "TFMBartPreTrainedModel",
    ]

try:
    if not is_flax_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    snake_case_ : List[str]    		=       [
        "FlaxMBartForConditionalGeneration",
        "FlaxMBartForQuestionAnswering",
        "FlaxMBartForSequenceClassification",
        "FlaxMBartModel",
        "FlaxMBartPreTrainedModel",
    ]


if TYPE_CHECKING:
    from .configuration_mbart import MBART_PRETRAINED_CONFIG_ARCHIVE_MAP, MBartConfig, MBartOnnxConfig

    try:
        if not is_sentencepiece_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_mbart import MBartTokenizer

    try:
        if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_mbart_fast import MBartTokenizerFast

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_mbart import (
            MBART_PRETRAINED_MODEL_ARCHIVE_LIST,
            MBartForCausalLM,
            MBartForConditionalGeneration,
            MBartForQuestionAnswering,
            MBartForSequenceClassification,
            MBartModel,
            MBartPreTrainedModel,
        )

    try:
        if not is_tf_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_tf_mbart import TFMBartForConditionalGeneration, TFMBartModel, TFMBartPreTrainedModel

    try:
        if not is_flax_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_flax_mbart import (
            FlaxMBartForConditionalGeneration,
            FlaxMBartForQuestionAnswering,
            FlaxMBartForSequenceClassification,
            FlaxMBartModel,
            FlaxMBartPreTrainedModel,
        )

else:
    import sys

    snake_case_ : Any    		=       _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)






