






"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


_snake_case          =   logging.get_logger(__name__)

_snake_case          =   {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_snake_case          =   [
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

_snake_case          =   {
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
_snake_case          =   {F'''funnel-transformer/{name}''': 512 for name in _model_names}
_snake_case          =   {F'''funnel-transformer/{name}''': {'do_lower_case': True} for name in _model_names}






class        UpperCamelCase			(	__snake_case      ):
 UpperCamelCase					:       Dict			=					VOCAB_FILES_NAMES
 UpperCamelCase					:       Tuple			=					PRETRAINED_VOCAB_FILES_MAP
 UpperCamelCase					:       Tuple			=					PRETRAINED_INIT_CONFIGURATION
 UpperCamelCase					:       Optional[int]			=					FunnelTokenizer
 UpperCamelCase					:       Any			=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCamelCase					:       Optional[Any]			=					2


 def __init__(     self		:						Any							,						UpperCAmelCase__		:						int=None							,						UpperCAmelCase__		:						Dict=None							,						UpperCAmelCase__		:						Optional[int]=True							,						UpperCAmelCase__		:						List[str]="<unk>"							,						UpperCAmelCase__		:						List[str]="<sep>"							,						UpperCAmelCase__		:						Any="<pad>"							,						UpperCAmelCase__		:						Any="<cls>"							,						UpperCAmelCase__		:						Union[str, Any]="<mask>"							,						UpperCAmelCase__		:						Tuple="<s>"							,						UpperCAmelCase__		:						Union[str, Any]="</s>"							,						UpperCAmelCase__		:						Optional[Any]=True							,						UpperCAmelCase__		:						Dict=True							,						UpperCAmelCase__		:						Dict=None							,						UpperCAmelCase__		:						List[str]="##"							,						**UpperCAmelCase__		:						Any							,						)					->       Tuple:
   super().__init__(
       UpperCAmelCase__							,						tokenizer_file=UpperCAmelCase__							,						do_lower_case=UpperCAmelCase__							,						unk_token=UpperCAmelCase__							,						sep_token=UpperCAmelCase__							,						pad_token=UpperCAmelCase__							,						cls_token=UpperCAmelCase__							,						mask_token=UpperCAmelCase__							,						bos_token=UpperCAmelCase__							,						eos_token=UpperCAmelCase__							,						clean_text=UpperCAmelCase__							,						tokenize_chinese_chars=UpperCAmelCase__							,						strip_accents=UpperCAmelCase__							,						wordpieces_prefix=UpperCAmelCase__							,						**UpperCAmelCase__							,						)

   _a  :       Tuple             =	json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
   if (
       normalizer_state.get("""lowercase"""							,						UpperCAmelCase__							) != do_lower_case
       or normalizer_state.get("""strip_accents"""							,						UpperCAmelCase__							) != strip_accents
       or normalizer_state.get("""handle_chinese_chars"""							,						UpperCAmelCase__							) != tokenize_chinese_chars
   ):
     _a  :       List[Any]             =	getattr(UpperCAmelCase__							,						normalizer_state.pop("""type"""							)							)
     _a  :       Union[str, Any]             =	do_lower_case
     _a  :       Optional[int]             =	strip_accents
     _a  :       Union[str, Any]             =	tokenize_chinese_chars
     _a  :       Tuple             =	normalizer_class(**UpperCAmelCase__							)

   _a  :       int             =	do_lower_case


 def _lowercase	(     self		:						List[Any]							,						UpperCAmelCase__		:						List[Any]							,						UpperCAmelCase__		:						Union[str, Any]=None							)					->       Union[str, Any]:
   _a  :       Optional[Any]             =	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

   if token_ids_a:
     output += token_ids_a + [self.sep_token_id]

   return output


 def _lowercase	(     self		:						int							,						UpperCAmelCase__		:						Any							,						UpperCAmelCase__		:						Tuple = None							)					->       List[int]:
   _a  :       List[Any]             =	[self.sep_token_id]
   _a  :       Tuple             =	[self.cls_token_id]
   if token_ids_a is None:
     return len(cls							) * [self.cls_token_type_id] + len(token_ids_a + sep							) * [0]
   return len(cls							) * [self.cls_token_type_id] + len(token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







 def _lowercase	(     self		:						str							,						UpperCAmelCase__		:						List[str]							,						UpperCAmelCase__		:						Any = None							)					->       Tuple[str]:
   _a  :       Tuple             =	self._tokenizer.model.save(UpperCAmelCase__							,						name=UpperCAmelCase__							)
   return tuple(UpperCAmelCase__							)



'''simple docstring'''




import os
import sys
import unittest


lowercase			:				Dict    		=			os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, 'utils'))

import get_test_info  # noqa: E402
from get_test_info import (  # noqa: E402
    get_model_to_test_mapping,
    get_model_to_tester_mapping,
    get_test_to_tester_mapping,
)


lowercase			:				Any    		=			os.path.join('tests', 'models', 'bert', 'test_modeling_bert.py')
lowercase			:				Optional[int]    		=			os.path.join('tests', 'models', 'blip', 'test_modeling_blip.py')





class  A				(							unittest.TestCase						):





     def 				__lowerCAmelCase							(							self	) ->				List[str]:






         """simple docstring"""






         A       :   Tuple					=						get_test_to_tester_mapping(SCREAMING_SNAKE_CASE	)
         A       :   Any					=						get_test_to_tester_mapping(SCREAMING_SNAKE_CASE	)

         A       :   List[Any]					=						{'''BertModelTest''': '''BertModelTester'''}

         A       :   int					=						{
             '''BlipModelTest''': '''BlipModelTester''',
             '''BlipTextImageModelTest''': '''BlipTextImageModelsModelTester''',
             '''BlipTextModelTest''': '''BlipTextModelTester''',
             '''BlipTextRetrievalModelTest''': '''BlipTextRetrievalModelTester''',
             '''BlipVQAModelTest''': '''BlipVQAModelTester''',
             '''BlipVisionModelTest''': '''BlipVisionModelTester''',
         }

         self.assertEqual(get_test_info.to_json(SCREAMING_SNAKE_CASE	)						,			SCREAMING_SNAKE_CASE	)
         self.assertEqual(get_test_info.to_json(SCREAMING_SNAKE_CASE	)						,			SCREAMING_SNAKE_CASE	)





     def 				__lowerCAmelCase							(							self	) ->				Any:






         """simple docstring"""






         A       :   Tuple					=						get_model_to_test_mapping(SCREAMING_SNAKE_CASE	)
         A       :   Tuple					=						get_model_to_test_mapping(SCREAMING_SNAKE_CASE	)

         A       :   List[str]					=						{
             '''BertForMaskedLM''': ['''BertModelTest'''],
             '''BertForMultipleChoice''': ['''BertModelTest'''],
             '''BertForNextSentencePrediction''': ['''BertModelTest'''],
             '''BertForPreTraining''': ['''BertModelTest'''],
             '''BertForQuestionAnswering''': ['''BertModelTest'''],
             '''BertForSequenceClassification''': ['''BertModelTest'''],
             '''BertForTokenClassification''': ['''BertModelTest'''],
             '''BertLMHeadModel''': ['''BertModelTest'''],
             '''BertModel''': ['''BertModelTest'''],
         }

         A       :   Union[str, Any]					=						{
             '''BlipForConditionalGeneration''': ['''BlipTextImageModelTest'''],
             '''BlipForImageTextRetrieval''': ['''BlipTextRetrievalModelTest'''],
             '''BlipForQuestionAnswering''': ['''BlipVQAModelTest'''],
             '''BlipModel''': ['''BlipModelTest'''],
             '''BlipTextModel''': ['''BlipTextModelTest'''],
             '''BlipVisionModel''': ['''BlipVisionModelTest'''],
         }

         self.assertEqual(get_test_info.to_json(SCREAMING_SNAKE_CASE	)						,			SCREAMING_SNAKE_CASE	)
         self.assertEqual(get_test_info.to_json(SCREAMING_SNAKE_CASE	)						,			SCREAMING_SNAKE_CASE	)





     def 				__lowerCAmelCase							(							self	) ->				Optional[int]:






         """simple docstring"""






         A       :   int					=						get_model_to_tester_mapping(SCREAMING_SNAKE_CASE	)
         A       :   Union[str, Any]					=						get_model_to_tester_mapping(SCREAMING_SNAKE_CASE	)

         A       :   Dict					=						{
             '''BertForMaskedLM''': ['''BertModelTester'''],
             '''BertForMultipleChoice''': ['''BertModelTester'''],
             '''BertForNextSentencePrediction''': ['''BertModelTester'''],
             '''BertForPreTraining''': ['''BertModelTester'''],
             '''BertForQuestionAnswering''': ['''BertModelTester'''],
             '''BertForSequenceClassification''': ['''BertModelTester'''],
             '''BertForTokenClassification''': ['''BertModelTester'''],
             '''BertLMHeadModel''': ['''BertModelTester'''],
             '''BertModel''': ['''BertModelTester'''],
         }

         A       :   str					=						{
             '''BlipForConditionalGeneration''': ['''BlipTextImageModelsModelTester'''],
             '''BlipForImageTextRetrieval''': ['''BlipTextRetrievalModelTester'''],
             '''BlipForQuestionAnswering''': ['''BlipVQAModelTester'''],
             '''BlipModel''': ['''BlipModelTester'''],
             '''BlipTextModel''': ['''BlipTextModelTester'''],
             '''BlipVisionModel''': ['''BlipVisionModelTester'''],
         }

         self.assertEqual(get_test_info.to_json(SCREAMING_SNAKE_CASE	)						,			SCREAMING_SNAKE_CASE	)
         self.assertEqual(get_test_info.to_json(SCREAMING_SNAKE_CASE	)						,			SCREAMING_SNAKE_CASE	)




