

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


lowerCAmelCase							:  Tuple   	=						logging.get_logger(__name__)

lowerCAmelCase							:  str   	=						{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCAmelCase							:  List[str]   	=						[
    """small""",
    """small-base""",
    """medium""",
    """medium-base""",
    """intermediate""",
    """intermediate-base""",
    """large""",
    """large-base""",
    """xlarge""",
    """xlarge-base""",
]

lowerCAmelCase							:  int   	=						{
    """vocab_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt""",
        """funnel-transformer/small-base""": """https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt""",
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt""",
        """funnel-transformer/large-base""": """https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json""",
        """funnel-transformer/small-base""": (
            """https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json""",
        """funnel-transformer/large-base""": (
            """https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"""
        ),
    },
}
lowerCAmelCase							:  List[Any]   	=						{f'''funnel-transformer/{name}''': 512 for name in _model_names}
lowerCAmelCase							:  Optional[Any]   	=						{f'''funnel-transformer/{name}''': {"""do_lower_case""": True} for name in _model_names}



class   __lowercase				( UpperCAmelCase_	):




			"""simple docstring"""







			_UpperCAmelCase   :							str										=     VOCAB_FILES_NAMES
			_UpperCAmelCase   :							str										=     PRETRAINED_VOCAB_FILES_MAP
			_UpperCAmelCase   :							Dict										=     PRETRAINED_INIT_CONFIGURATION
			_UpperCAmelCase   :							Any										=     FunnelTokenizer
			_UpperCAmelCase   :							Dict										=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCAmelCase   :							int										=     2



			def __init__( self  :     Union[str, Any]   ,						lowerCAmelCase__  :     int=None   ,						lowerCAmelCase__  :     Optional[Any]=None   ,						lowerCAmelCase__  :     Union[str, Any]=True   ,						lowerCAmelCase__  :     Optional[int]="<unk>"   ,						lowerCAmelCase__  :     Tuple="<sep>"   ,						lowerCAmelCase__  :     Optional[Any]="<pad>"   ,						lowerCAmelCase__  :     Optional[int]="<cls>"   ,						lowerCAmelCase__  :     List[str]="<mask>"   ,						lowerCAmelCase__  :     Optional[Any]="<s>"   ,						lowerCAmelCase__  :     Optional[int]="</s>"   ,						lowerCAmelCase__  :     Optional[int]=True   ,						lowerCAmelCase__  :     List[str]=True   ,						lowerCAmelCase__  :     Union[str, Any]=None   ,						lowerCAmelCase__  :     Tuple="##"   ,						**lowerCAmelCase__  :     Union[str, Any]   ,						):
									super().__init__(
									    lowerCAmelCase__   ,						tokenizer_file=lowerCAmelCase__   ,						do_lower_case=lowerCAmelCase__   ,						unk_token=lowerCAmelCase__   ,						sep_token=lowerCAmelCase__   ,						pad_token=lowerCAmelCase__   ,						cls_token=lowerCAmelCase__   ,						mask_token=lowerCAmelCase__   ,						bos_token=lowerCAmelCase__   ,						eos_token=lowerCAmelCase__   ,						clean_text=lowerCAmelCase__   ,						tokenize_chinese_chars=lowerCAmelCase__   ,						strip_accents=lowerCAmelCase__   ,						wordpieces_prefix=lowerCAmelCase__   ,						**lowerCAmelCase__   ,						)

									SCREAMING_SNAKE_CASE_: int  		=       json.loads(self.backend_tokenizer.normalizer.__getstate__())
									if (
									    normalizer_state.get("lowercase"   ,						lowerCAmelCase__) != do_lower_case
									    or normalizer_state.get("strip_accents"   ,						lowerCAmelCase__) != strip_accents
									    or normalizer_state.get("handle_chinese_chars"   ,						lowerCAmelCase__) != tokenize_chinese_chars
									):
															SCREAMING_SNAKE_CASE_: str  		=       getattr(lowerCAmelCase__   ,						normalizer_state.pop("type"))
															SCREAMING_SNAKE_CASE_: Tuple  		=       do_lower_case
															SCREAMING_SNAKE_CASE_: str  		=       strip_accents
															SCREAMING_SNAKE_CASE_: List[Any]  		=       tokenize_chinese_chars
															SCREAMING_SNAKE_CASE_: Optional[int]  		=       normalizer_class(**lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_: Dict  		=       do_lower_case



			def        _SCREAMING_SNAKE_CASE			( self  :     List[Any]   ,						lowerCAmelCase__  :     List[str]   ,						lowerCAmelCase__  :     str=None):
									SCREAMING_SNAKE_CASE_: int  		=       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
															output += token_ids_a + [self.sep_token_id]

									return output



			def        _SCREAMING_SNAKE_CASE			( self  :     int   ,						lowerCAmelCase__  :     List[int]   ,						lowerCAmelCase__  :     Optional[List[int]] = None):
									SCREAMING_SNAKE_CASE_: List[str]  		=       [self.sep_token_id]
									SCREAMING_SNAKE_CASE_: List[Any]  		=       [self.cls_token_id]
									if token_ids_a is None:
															return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0]
									return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]




			def        _SCREAMING_SNAKE_CASE			( self  :     Any   ,						lowerCAmelCase__  :     str   ,						lowerCAmelCase__  :     Optional[str] = None):
									SCREAMING_SNAKE_CASE_: str  		=       self._tokenizer.model.save(lowerCAmelCase__   ,						name=lowerCAmelCase__)
									return tuple(lowerCAmelCase__)









from ...utils import (
    OptionalDependencyNotAvailable,
    is_torch_available,
    is_transformers_available,
    is_transformers_version,
)


try:
						if not (is_transformers_available() and is_torch_available() and is_transformers_version(""">=""", """4.25.0""")):
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						from ...utils.dummy_torch_and_transformers_objects import UnCLIPImageVariationPipeline, UnCLIPPipeline
else:
						from .pipeline_unclip import UnCLIPPipeline
						from .pipeline_unclip_image_variation import UnCLIPImageVariationPipeline
						from .text_proj import UnCLIPTextProjModel






