

"""simple docstring"""





import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class 				SCREAMING_SNAKE_CASE					(		a_    ,							unittest.TestCase       ):






					"""simple docstring"""



					lowercase__				 =       MobileBertTokenizer
					lowercase__				 =       MobileBertTokenizerFast
					lowercase__				 =       True
					lowercase__				 =       True
					lowercase__				 =       filter_non_english
					lowercase__				 =       "google/mobilebert-uncased"







					def        __lowerCAmelCase (       self			:				Any		):
						super().setUp()

						lowerCAmelCase__	:						Tuple  				=	[
						    '''[UNK]''',
						    '''[CLS]''',
						    '''[SEP]''',
						    '''[PAD]''',
						    '''[MASK]''',
						    '''want''',
						    '''##want''',
						    '''##ed''',
						    '''wa''',
						    '''un''',
						    '''runn''',
						    '''##ing''',
						    ''',''',
						    '''low''',
						    '''lowest''',
						]
						lowerCAmelCase__	:						List[Any]  				=	os.path.join(self.tmpdirname   ,VOCAB_FILES_NAMES['''vocab_file''']		)
						with open(self.vocab_file   ,'''w'''   ,encoding='''utf-8'''		) as vocab_writer:
							vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]		)		)

						lowerCAmelCase__	:						str  				=	[
						    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
						    for tokenizer_def in self.tokenizers_list
						]







					def        __lowerCAmelCase (       self			:				str   ,lowercase_			:				Optional[int]		):
						lowerCAmelCase__	:						Optional[int]  				=	'''UNwant\u00E9d,running'''
						lowerCAmelCase__	:						Tuple  				=	'''unwanted, running'''
						return input_text, output_text







					def        __lowerCAmelCase (       self			:				Optional[Any]		):
						lowerCAmelCase__	:						Optional[Any]  				=	self.tokenizer_class(self.vocab_file		)

						lowerCAmelCase__	:						int  				=	tokenizer.tokenize('''UNwant\u00E9d,running'''		)
						self.assertListEqual(lowercase_   ,['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']		)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_		)   ,[9, 6, 7, 1_2, 1_0, 1_1]		)







					def        __lowerCAmelCase (       self			:				Optional[Any]		):
						if not self.test_rust_tokenizer:
							return

						lowerCAmelCase__	:						List[Any]  				=	self.get_tokenizer()
						lowerCAmelCase__	:						Dict  				=	self.get_rust_tokenizer()

						lowerCAmelCase__	:						str  				=	'''UNwant\u00E9d,running'''

						lowerCAmelCase__	:						Dict  				=	tokenizer.tokenize(lowercase_		)
						lowerCAmelCase__	:						str  				=	rust_tokenizer.tokenize(lowercase_		)
						self.assertListEqual(lowercase_   ,lowercase_		)

						lowerCAmelCase__	:						List[str]  				=	tokenizer.encode(lowercase_   ,add_special_tokens=lowercase_		)
						lowerCAmelCase__	:						Any  				=	rust_tokenizer.encode(lowercase_   ,add_special_tokens=lowercase_		)
						self.assertListEqual(lowercase_   ,lowercase_		)

						lowerCAmelCase__	:						Union[str, Any]  				=	self.get_rust_tokenizer()
						lowerCAmelCase__	:						str  				=	tokenizer.encode(lowercase_		)
						lowerCAmelCase__	:						Optional[int]  				=	rust_tokenizer.encode(lowercase_		)
						self.assertListEqual(lowercase_   ,lowercase_		)

						# With lower casing
						lowerCAmelCase__	:						int  				=	self.get_tokenizer(do_lower_case=lowercase_		)
						lowerCAmelCase__	:						Optional[Any]  				=	self.get_rust_tokenizer(do_lower_case=lowercase_		)

						lowerCAmelCase__	:						Optional[Any]  				=	'''UNwant\u00E9d,running'''

						lowerCAmelCase__	:						List[Any]  				=	tokenizer.tokenize(lowercase_		)
						lowerCAmelCase__	:						Any  				=	rust_tokenizer.tokenize(lowercase_		)
						self.assertListEqual(lowercase_   ,lowercase_		)

						lowerCAmelCase__	:						Union[str, Any]  				=	tokenizer.encode(lowercase_   ,add_special_tokens=lowercase_		)
						lowerCAmelCase__	:						Tuple  				=	rust_tokenizer.encode(lowercase_   ,add_special_tokens=lowercase_		)
						self.assertListEqual(lowercase_   ,lowercase_		)

						lowerCAmelCase__	:						int  				=	self.get_rust_tokenizer()
						lowerCAmelCase__	:						str  				=	tokenizer.encode(lowercase_		)
						lowerCAmelCase__	:						List[str]  				=	rust_tokenizer.encode(lowercase_		)
						self.assertListEqual(lowercase_   ,lowercase_		)







					def        __lowerCAmelCase (       self			:				Any		):
						lowerCAmelCase__	:						int  				=	BasicTokenizer()

						self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''		)   ,['''ah''', '''\u535A''', '''\u63A8''', '''zz''']		)







					def        __lowerCAmelCase (       self			:				Optional[int]		):
						lowerCAmelCase__	:						List[str]  				=	BasicTokenizer(do_lower_case=lowercase_		)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''		)   ,['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']		)
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)   ,['''hello''']		)







					def        __lowerCAmelCase (       self			:				Optional[int]		):
						lowerCAmelCase__	:						Union[str, Any]  				=	BasicTokenizer(do_lower_case=lowercase_   ,strip_accents=lowercase_		)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)   ,['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']		)
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)   ,['''h\u00E9llo''']		)







					def        __lowerCAmelCase (       self			:				List[str]		):
						lowerCAmelCase__	:						Any  				=	BasicTokenizer(do_lower_case=lowercase_   ,strip_accents=lowercase_		)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)   ,['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']		)
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)   ,['''hello''']		)







					def        __lowerCAmelCase (       self			:				int		):
						lowerCAmelCase__	:						Optional[Any]  				=	BasicTokenizer(do_lower_case=lowercase_		)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)   ,['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']		)
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)   ,['''hello''']		)







					def        __lowerCAmelCase (       self			:				List[Any]		):
						lowerCAmelCase__	:						Optional[Any]  				=	BasicTokenizer(do_lower_case=lowercase_		)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''		)   ,['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']		)







					def        __lowerCAmelCase (       self			:				List[Any]		):
						lowerCAmelCase__	:						Union[str, Any]  				=	BasicTokenizer(do_lower_case=lowercase_   ,strip_accents=lowercase_		)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)   ,['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']		)







					def        __lowerCAmelCase (       self			:				List[Any]		):
						lowerCAmelCase__	:						Optional[int]  				=	BasicTokenizer(do_lower_case=lowercase_   ,strip_accents=lowercase_		)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)   ,['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']		)







					def        __lowerCAmelCase (       self			:				List[Any]		):
						lowerCAmelCase__	:						Tuple  				=	BasicTokenizer(do_lower_case=lowercase_   ,never_split=['''[UNK]''']		)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''		)   ,['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']		)







					def        __lowerCAmelCase (       self			:				Any		):
						lowerCAmelCase__	:						Any  				=	['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

						lowerCAmelCase__	:						Tuple  				=	{}
						for i, token in enumerate(lowercase_		):
							lowerCAmelCase__	:						Optional[Any]  				=	i
						lowerCAmelCase__	:						List[Any]  				=	WordpieceTokenizer(vocab=lowercase_   ,unk_token='''[UNK]'''		)

						self.assertListEqual(tokenizer.tokenize(''''''		)   ,[]		)

						self.assertListEqual(tokenizer.tokenize('''unwanted running'''		)   ,['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']		)

						self.assertListEqual(tokenizer.tokenize('''unwantedX running'''		)   ,['''[UNK]''', '''runn''', '''##ing''']		)







					def        __lowerCAmelCase (       self			:				Dict		):
						self.assertTrue(_is_whitespace(''' '''		)		)
						self.assertTrue(_is_whitespace('''\t'''		)		)
						self.assertTrue(_is_whitespace('''\r'''		)		)
						self.assertTrue(_is_whitespace('''\n'''		)		)
						self.assertTrue(_is_whitespace('''\u00A0'''		)		)

						self.assertFalse(_is_whitespace('''A'''		)		)
						self.assertFalse(_is_whitespace('''-'''		)		)







					def        __lowerCAmelCase (       self			:				List[Any]		):
						self.assertTrue(_is_control('''\u0005'''		)		)

						self.assertFalse(_is_control('''A'''		)		)
						self.assertFalse(_is_control(''' '''		)		)
						self.assertFalse(_is_control('''\t'''		)		)
						self.assertFalse(_is_control('''\r'''		)		)







					def        __lowerCAmelCase (       self			:				Any		):
						self.assertTrue(_is_punctuation('''-'''		)		)
						self.assertTrue(_is_punctuation('''$'''		)		)
						self.assertTrue(_is_punctuation('''`'''		)		)
						self.assertTrue(_is_punctuation('''.'''		)		)

						self.assertFalse(_is_punctuation('''A'''		)		)
						self.assertFalse(_is_punctuation(''' '''		)		)







					def        __lowerCAmelCase (       self			:				Optional[Any]		):
						lowerCAmelCase__	:						Any  				=	self.get_tokenizer()
						lowerCAmelCase__	:						Union[str, Any]  				=	self.get_rust_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(lowercase_		) for t in ['''Test''', '''\xad''', '''test''']]   ,[['''[UNK]'''], [], ['''[UNK]''']]		)

						self.assertListEqual(
						    [rust_tokenizer.tokenize(lowercase_		) for t in ['''Test''', '''\xad''', '''test''']]   ,[['''[UNK]'''], [], ['''[UNK]''']]		)







					@slow
					def        __lowerCAmelCase (       self			:				List[Any]		):
						lowerCAmelCase__	:						int  				=	self.tokenizer_class.from_pretrained('''google/mobilebert-uncased'''		)

						lowerCAmelCase__	:						Union[str, Any]  				=	tokenizer.encode('''sequence builders'''   ,add_special_tokens=lowercase_		)
						lowerCAmelCase__	:						Any  				=	tokenizer.encode('''multi-sequence build'''   ,add_special_tokens=lowercase_		)

						lowerCAmelCase__	:						Optional[int]  				=	tokenizer.build_inputs_with_special_tokens(lowercase_		)
						lowerCAmelCase__	:						List[str]  				=	tokenizer.build_inputs_with_special_tokens(lowercase_   ,lowercase_		)

						assert encoded_sentence == [1_0_1] + text + [1_0_2]
						assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]







					def        __lowerCAmelCase (       self			:				str		):
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'		):
								lowerCAmelCase__	:						Optional[Any]  				=	self.rust_tokenizer_class.from_pretrained(lowercase_   ,**lowercase_		)

								lowerCAmelCase__	:						List[str]  				=	F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
								lowerCAmelCase__	:						Union[str, Any]  				=	tokenizer_r.encode_plus(
								    lowercase_   ,return_attention_mask=lowercase_   ,return_token_type_ids=lowercase_   ,return_offsets_mapping=lowercase_   ,add_special_tokens=lowercase_   ,)

								lowerCAmelCase__	:						List[Any]  				=	tokenizer_r.do_lower_case if hasattr(lowercase_   ,'''do_lower_case'''		) else False
								lowerCAmelCase__	:						Optional[Any]  				=	(
								    [
								        ((0, 0), tokenizer_r.cls_token),
								        ((0, 1), '''A'''),
								        ((1, 2), ''','''),
								        ((3, 5), '''na'''),
								        ((5, 6), '''##ï'''),
								        ((6, 8), '''##ve'''),
								        ((9, 1_5), tokenizer_r.mask_token),
								        ((1_6, 2_1), '''Allen'''),
								        ((2_1, 2_3), '''##NL'''),
								        ((2_3, 2_4), '''##P'''),
								        ((2_5, 3_3), '''sentence'''),
								        ((3_3, 3_4), '''.'''),
								        ((0, 0), tokenizer_r.sep_token),
								    ]
								    if not do_lower_case
								    else [
								        ((0, 0), tokenizer_r.cls_token),
								        ((0, 1), '''a'''),
								        ((1, 2), ''','''),
								        ((3, 8), '''naive'''),
								        ((9, 1_5), tokenizer_r.mask_token),
								        ((1_6, 2_1), '''allen'''),
								        ((2_1, 2_3), '''##nl'''),
								        ((2_3, 2_4), '''##p'''),
								        ((2_5, 3_3), '''sentence'''),
								        ((3_3, 3_4), '''.'''),
								        ((0, 0), tokenizer_r.sep_token),
								    ]
								)

								self.assertEqual(
								    [e[1] for e in expected_results]   ,tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']		)		)
								self.assertEqual([e[0] for e in expected_results]   ,tokens['''offset_mapping''']		)







					def        __lowerCAmelCase (       self			:				Tuple		):
						lowerCAmelCase__	:						Union[str, Any]  				=	['''的''', '''人''', '''有''']
						lowerCAmelCase__	:						Optional[Any]  				=	''''''.join(lowercase_		)
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'		):
								lowerCAmelCase__	:						Dict  				=	True
								lowerCAmelCase__	:						Union[str, Any]  				=	self.tokenizer_class.from_pretrained(lowercase_   ,**lowercase_		)
								lowerCAmelCase__	:						Union[str, Any]  				=	self.rust_tokenizer_class.from_pretrained(lowercase_   ,**lowercase_		)

								lowerCAmelCase__	:						Dict  				=	tokenizer_p.encode(lowercase_   ,add_special_tokens=lowercase_		)
								lowerCAmelCase__	:						Tuple  				=	tokenizer_r.encode(lowercase_   ,add_special_tokens=lowercase_		)

								lowerCAmelCase__	:						Tuple  				=	tokenizer_r.convert_ids_to_tokens(lowercase_		)
								lowerCAmelCase__	:						Tuple  				=	tokenizer_p.convert_ids_to_tokens(lowercase_		)

								# it is expected that each Chinese character is not preceded by "##"
								self.assertListEqual(lowercase_   ,lowercase_		)
								self.assertListEqual(lowercase_   ,lowercase_		)

								lowerCAmelCase__	:						List[str]  				=	False
								lowerCAmelCase__	:						Dict  				=	self.rust_tokenizer_class.from_pretrained(lowercase_   ,**lowercase_		)
								lowerCAmelCase__	:						Tuple  				=	self.tokenizer_class.from_pretrained(lowercase_   ,**lowercase_		)

								lowerCAmelCase__	:						Any  				=	tokenizer_r.encode(lowercase_   ,add_special_tokens=lowercase_		)
								lowerCAmelCase__	:						List[str]  				=	tokenizer_p.encode(lowercase_   ,add_special_tokens=lowercase_		)

								lowerCAmelCase__	:						Any  				=	tokenizer_r.convert_ids_to_tokens(lowercase_		)
								lowerCAmelCase__	:						Dict  				=	tokenizer_p.convert_ids_to_tokens(lowercase_		)

								# it is expected that only the first Chinese character is not preceded by "##".
								lowerCAmelCase__	:						int  				=	[
								    F'##{token}' if idx != 0 else token for idx, token in enumerate(lowercase_		)
								]
								self.assertListEqual(lowercase_   ,lowercase_		)
								self.assertListEqual(lowercase_   ,lowercase_		)







def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	Optional[Any]    ,			SCREAMING_SNAKE_CASE__			:	Union[str, Any]      )							->      Tuple:





    '''simple docstring'''






    A__          =   0
    A__          =   len(SCREAMING_SNAKE_CASE__      ) - 1

    while left <= right:
        # avoid divided by 0 during interpolation
        if sorted_collection[left] == sorted_collection[right]:
            if sorted_collection[left] == item:
                return left
            else:
                return None

        A__          =   left + ((item - sorted_collection[left]) * (right - left)) // (
            sorted_collection[right] - sorted_collection[left]
        )

        # out of range check
        if point < 0 or point >= len(SCREAMING_SNAKE_CASE__      ):
            return None

        A__          =   sorted_collection[point]
        if current_item == item:
            return point
        else:
            if point < left:
                A__          =   left
                A__          =   point
            elif point > right:
                A__          =   right
                A__          =   point
            else:
                if item < current_item:
                    A__          =   point - 1
                else:
                    A__          =   point + 1
    return None
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	Optional[Any]    ,			SCREAMING_SNAKE_CASE__			:	List[str]    ,			SCREAMING_SNAKE_CASE__			:	Optional[Any]    ,			SCREAMING_SNAKE_CASE__			:	List[str]      )							->      str:





    '''simple docstring'''






    if sorted_collection[left] == sorted_collection[right]:
        if sorted_collection[left] == item:
            return left
        else:
            return None

    A__          =   left + ((item - sorted_collection[left]) * (right - left)) // (
        sorted_collection[right] - sorted_collection[left]
    )

    # out of range check
    if point < 0 or point >= len(SCREAMING_SNAKE_CASE__      ):
        return None

    if sorted_collection[point] == item:
        return point
    elif point < left:
        return interpolation_search_by_recursion(SCREAMING_SNAKE_CASE__    ,			SCREAMING_SNAKE_CASE__    ,			SCREAMING_SNAKE_CASE__    ,			SCREAMING_SNAKE_CASE__      )
    elif point > right:
        return interpolation_search_by_recursion(SCREAMING_SNAKE_CASE__    ,			SCREAMING_SNAKE_CASE__    ,			SCREAMING_SNAKE_CASE__    ,			SCREAMING_SNAKE_CASE__      )
    else:
        if sorted_collection[point] > item:
            return interpolation_search_by_recursion(
                SCREAMING_SNAKE_CASE__    ,			SCREAMING_SNAKE_CASE__    ,			SCREAMING_SNAKE_CASE__    ,			point - 1      )
        else:
            return interpolation_search_by_recursion(
                SCREAMING_SNAKE_CASE__    ,			SCREAMING_SNAKE_CASE__    ,			point + 1    ,			SCREAMING_SNAKE_CASE__      )
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	Tuple      )							->      Tuple:





    '''simple docstring'''






    if collection != sorted(SCREAMING_SNAKE_CASE__      ):
        raise ValueError('Collection must be ascending sorted'      )
    return True


if __name__ == "__main__":
     import sys
     lowercase_       		=  0
     if debug == 1:
          lowercase_       		=  [10, 30, 40, 45, 50, 66, 77, 93]
          try:
               __assert_sorted(collection)
          except ValueError:
               sys.exit("Sequence must be ascending sorted to apply interpolation search")
          lowercase_       		=  67

     lowercase_       		=  interpolation_search(collection, target)
     if result is not None:
          print(f"""{target} found at positions: {result}""")
     else:
          print("Not found")

