
"""simple docstring"""


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


lowerCAmelCase								=						logging.get_logger(__name__)

lowerCAmelCase								=						{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCAmelCase								=						[
    """small""",
    """small-base""",
    """medium""",
    """medium-base""",
    """intermediate""",
    """intermediate-base""",
    """large""",
    """large-base""",
    """xlarge""",
    """xlarge-base""",
]

lowerCAmelCase								=						{
    """vocab_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt""",
        """funnel-transformer/small-base""": """https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt""",
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt""",
        """funnel-transformer/large-base""": """https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json""",
        """funnel-transformer/small-base""": (
            """https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json""",
        """funnel-transformer/large-base""": (
            """https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"""
        ),
    },
}
lowerCAmelCase								=						{f"""funnel-transformer/{name}""": 5_12 for name in _model_names}
lowerCAmelCase								=						{f"""funnel-transformer/{name}""": {"""do_lower_case""": True} for name in _model_names}

class     A_	(     A_   ):



							"""simple docstring"""






							SCREAMING_SNAKE_CASE_	      =   VOCAB_FILES_NAMES
							SCREAMING_SNAKE_CASE_	      =   PRETRAINED_VOCAB_FILES_MAP
							SCREAMING_SNAKE_CASE_	      =   PRETRAINED_INIT_CONFIGURATION
							SCREAMING_SNAKE_CASE_	      =   FunnelTokenizer
							SCREAMING_SNAKE_CASE_	      =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							SCREAMING_SNAKE_CASE_	      =   2





							def __init__(  self  :Optional[int]     ,							lowerCamelCase_  :Dict=None     ,							lowerCamelCase_  :List[str]=None     ,							lowerCamelCase_  :Optional[Any]=True     ,							lowerCamelCase_  :Optional[int]="<unk>"     ,							lowerCamelCase_  :Dict="<sep>"     ,							lowerCamelCase_  :Any="<pad>"     ,							lowerCamelCase_  :str="<cls>"     ,							lowerCamelCase_  :Optional[Any]="<mask>"     ,							lowerCamelCase_  :int="<s>"     ,							lowerCamelCase_  :Dict="</s>"     ,							lowerCamelCase_  :Optional[int]=True     ,							lowerCamelCase_  :List[str]=True     ,							lowerCamelCase_  :Dict=None     ,							lowerCamelCase_  :str="##"     ,							**lowerCamelCase_  :Optional[Any]     ,							):





										"""simple docstring"""

										super().__init__(
										    _snake_case     ,							tokenizer_file=_snake_case     ,							do_lower_case=_snake_case     ,							unk_token=_snake_case     ,							sep_token=_snake_case     ,							pad_token=_snake_case     ,							cls_token=_snake_case     ,							mask_token=_snake_case     ,							bos_token=_snake_case     ,							eos_token=_snake_case     ,							clean_text=_snake_case     ,							tokenize_chinese_chars=_snake_case     ,							strip_accents=_snake_case     ,							wordpieces_prefix=_snake_case     ,							**_snake_case     ,							)

										lowerCamelCase__	:     Union[str, Any]              =json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
										if (
										    normalizer_state.get('lowercase'     ,							_snake_case       ) != do_lower_case
										    or normalizer_state.get('strip_accents'     ,							_snake_case       ) != strip_accents
										    or normalizer_state.get('handle_chinese_chars'     ,							_snake_case       ) != tokenize_chinese_chars
										):
													lowerCamelCase__	:     List[str]              =getattr(_snake_case     ,							normalizer_state.pop('type'       )       )
													lowerCamelCase__	:     List[str]              =do_lower_case
													lowerCamelCase__	:     Any              =strip_accents
													lowerCamelCase__	:     Union[str, Any]              =tokenize_chinese_chars
													lowerCamelCase__	:     Union[str, Any]              =normalizer_class(**_snake_case       )

										lowerCamelCase__	:     List[str]              =do_lower_case





							def       UpperCAmelCase__							(  self  :int     ,							lowerCamelCase_  :Tuple     ,							lowerCamelCase_  :Tuple=None       ):





										"""simple docstring"""

										lowerCamelCase__	:     Any              =[self.cls_token_id] + token_ids_a + [self.sep_token_id]

										if token_ids_a:
													output += token_ids_a + [self.sep_token_id]

										return output





							def       UpperCAmelCase__							(  self  :Tuple     ,							lowerCamelCase_  :List[int]     ,							lowerCamelCase_  :Optional[List[int]] = None       ):





										"""simple docstring"""

										lowerCamelCase__	:     Optional[int]              =[self.sep_token_id]
										lowerCamelCase__	:     List[Any]              =[self.cls_token_id]
										if token_ids_a is None:
													return len(cls       ) * [self.cls_token_type_id] + len(token_ids_a + sep       ) * [0]
										return len(cls       ) * [self.cls_token_type_id] + len(token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]





							def       UpperCAmelCase__							(  self  :Tuple     ,							lowerCamelCase_  :str     ,							lowerCamelCase_  :Optional[str] = None       ):





										"""simple docstring"""

										lowerCamelCase__	:     str              =self._tokenizer.model.save(_snake_case     ,							name=_snake_case       )
										return tuple(_snake_case       )




"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCAmelCase_   						=   {'configuration_xglm': ['XGLM_PRETRAINED_CONFIG_ARCHIVE_MAP', 'XGLMConfig']}

try:
							if not is_sentencepiece_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowerCAmelCase_   						=   ['XGLMTokenizer']

try:
							if not is_tokenizers_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowerCAmelCase_   						=   ['XGLMTokenizerFast']

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowerCAmelCase_   						=   [
							    'XGLM_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'XGLMForCausalLM',
							    'XGLMModel',
							    'XGLMPreTrainedModel',
							]


try:
							if not is_flax_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowerCAmelCase_   						=   [
							    'FlaxXGLMForCausalLM',
							    'FlaxXGLMModel',
							    'FlaxXGLMPreTrainedModel',
							]


try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowerCAmelCase_   						=   [
							    'TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'TFXGLMForCausalLM',
							    'TFXGLMModel',
							    'TFXGLMPreTrainedModel',
							]


if TYPE_CHECKING:
							from .configuration_xglm import XGLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XGLMConfig

							try:
														if not is_sentencepiece_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_xglm import XGLMTokenizer

							try:
														if not is_tokenizers_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_xglm_fast import XGLMTokenizerFast

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_xglm import XGLM_PRETRAINED_MODEL_ARCHIVE_LIST, XGLMForCausalLM, XGLMModel, XGLMPreTrainedModel

							try:
														if not is_flax_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_flax_xglm import FlaxXGLMForCausalLM, FlaxXGLMModel, FlaxXGLMPreTrainedModel

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_xglm import (
														    TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFXGLMForCausalLM,
														    TFXGLMModel,
														    TFXGLMPreTrainedModel,
														)


else:
							import sys

							lowerCAmelCase_   						=   _LazyModule(__name__, globals()['__file__'], _import_structure)
