
'''simple docstring'''




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


lowercase			:				int    		=			logging.get_logger(__name__)

lowercase			:				List[Any]    		=			{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowercase			:				Dict    		=			[
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

lowercase			:				Dict    		=			{
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
lowercase			:				Union[str, Any]    		=			{f'''funnel-transformer/{name}''': 5_12 for name in _model_names}
lowercase			:				Optional[Any]    		=			{f'''funnel-transformer/{name}''': {'do_lower_case': True} for name in _model_names}





class  A				(							__snake_case						):
     __magic_name__						=		VOCAB_FILES_NAMES
     __magic_name__						=		PRETRAINED_VOCAB_FILES_MAP
     __magic_name__						=		PRETRAINED_INIT_CONFIGURATION
     __magic_name__						=		FunnelTokenizer
     __magic_name__						=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __magic_name__						=		2





     def __init__(							self						,			SCREAMING_SNAKE_CASE=None						,			SCREAMING_SNAKE_CASE=None						,			SCREAMING_SNAKE_CASE=True						,			SCREAMING_SNAKE_CASE="<unk>"						,			SCREAMING_SNAKE_CASE="<sep>"						,			SCREAMING_SNAKE_CASE="<pad>"						,			SCREAMING_SNAKE_CASE="<cls>"						,			SCREAMING_SNAKE_CASE="<mask>"						,			SCREAMING_SNAKE_CASE="<s>"						,			SCREAMING_SNAKE_CASE="</s>"						,			SCREAMING_SNAKE_CASE=True						,			SCREAMING_SNAKE_CASE=True						,			SCREAMING_SNAKE_CASE=None						,			SCREAMING_SNAKE_CASE="##"						,			**SCREAMING_SNAKE_CASE						,			) ->				Tuple:






         """simple docstring"""






         super().__init__(
             SCREAMING_SNAKE_CASE						,			tokenizer_file=SCREAMING_SNAKE_CASE						,			do_lower_case=SCREAMING_SNAKE_CASE						,			unk_token=SCREAMING_SNAKE_CASE						,			sep_token=SCREAMING_SNAKE_CASE						,			pad_token=SCREAMING_SNAKE_CASE						,			cls_token=SCREAMING_SNAKE_CASE						,			mask_token=SCREAMING_SNAKE_CASE						,			bos_token=SCREAMING_SNAKE_CASE						,			eos_token=SCREAMING_SNAKE_CASE						,			clean_text=SCREAMING_SNAKE_CASE						,			tokenize_chinese_chars=SCREAMING_SNAKE_CASE						,			strip_accents=SCREAMING_SNAKE_CASE						,			wordpieces_prefix=SCREAMING_SNAKE_CASE						,			**SCREAMING_SNAKE_CASE						,			)

         A       :   Tuple					=						json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
         if (
             normalizer_state.get('''lowercase'''						,			SCREAMING_SNAKE_CASE	) != do_lower_case
             or normalizer_state.get('''strip_accents'''						,			SCREAMING_SNAKE_CASE	) != strip_accents
             or normalizer_state.get('''handle_chinese_chars'''						,			SCREAMING_SNAKE_CASE	) != tokenize_chinese_chars
         ):
             A       :   List[Any]					=						getattr(SCREAMING_SNAKE_CASE						,			normalizer_state.pop('''type'''	)	)
             A       :   Union[str, Any]					=						do_lower_case
             A       :   Optional[int]					=						strip_accents
             A       :   Union[str, Any]					=						tokenize_chinese_chars
             A       :   Tuple					=						normalizer_class(**SCREAMING_SNAKE_CASE	)

         A       :   int					=						do_lower_case





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE=None	) ->				Union[str, Any]:






         """simple docstring"""






         A       :   Optional[Any]					=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
             output += token_ids_a + [self.sep_token_id]

         return output





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE = None	) ->				List[int]:






         """simple docstring"""






         A       :   List[Any]					=						[self.sep_token_id]
         A       :   Tuple					=						[self.cls_token_id]
         if token_ids_a is None:
             return len(cls	) * [self.cls_token_type_id] + len(token_ids_a + sep	) * [0]
         return len(cls	) * [self.cls_token_type_id] + len(token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE = None	) ->				Tuple[str]:






         """simple docstring"""






         A       :   Tuple					=						self._tokenizer.model.save(SCREAMING_SNAKE_CASE						,			name=SCREAMING_SNAKE_CASE	)
         return tuple(SCREAMING_SNAKE_CASE	)






'''simple docstring'''




import os







def 		lowerCAmelCase_				(   ):




    '''simple docstring'''



    A       :   List[Any]					=						os.path.join(os.path.dirname(snake_case__						)		,				'''num.txt'''						)
    with open(snake_case__						) as file_hand:
        return str(sum(int(snake_case__						) for line in file_hand						)						)[:10]


if __name__ == "__main__":
      print(solution())




