



'''simple docstring'''






import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 				SCREAMING_SNAKE_CASE(							A__							,		unittest.TestCase			):







	"""simple docstring"""






	lowerCamelCase__			 =       MobileBertTokenizer
	lowerCamelCase__			 =       MobileBertTokenizerFast
	lowerCamelCase__			 =       True
	lowerCamelCase__			 =       True
	lowerCamelCase__			 =       filter_non_english
	lowerCamelCase__			 =       """google/mobilebert-uncased"""







	def      A	(						self			:     Any		)			->		int:
								super().setUp()

								UpperCAmelCase :       str 							=		[
								    '''[UNK]''',
								    '''[CLS]''',
								    '''[SEP]''',
								    '''[PAD]''',
								    '''[MASK]''',
								    '''want''',
								    '''##want''',
								    '''##ed''',
								    '''wa''',
								    '''un''',
								    '''runn''',
								    '''##ing''',
								    ''',''',
								    '''low''',
								    '''lowest''',
								]
								UpperCAmelCase :       Dict 							=		os.path.join(self.tmpdirname		,			VOCAB_FILES_NAMES['''vocab_file''']		)
								with open(self.vocab_file		,			'''w'''		,			encoding='''utf-8'''		) as vocab_writer:
															vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]		)		)

								UpperCAmelCase :       List[Any] 							=		[
								    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
								    for tokenizer_def in self.tokenizers_list
								]







	def      A	(						self			:     Optional[Any]		,			__snake_case			:     Union[str, Any]		)			->		Optional[Any]:
								UpperCAmelCase :       Optional[Any] 							=		'''UNwant\u00E9d,running'''
								UpperCAmelCase :       int 							=		'''unwanted, running'''
								return input_text, output_text







	def      A	(						self			:     Optional[Any]		)			->		Dict:
								UpperCAmelCase :       List[str] 							=		self.tokenizer_class(self.vocab_file		)

								UpperCAmelCase :       Dict 							=		tokenizer.tokenize('''UNwant\u00E9d,running'''		)
								self.assertListEqual(__snake_case		,			['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']		)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case		)		,			[9, 6, 7, 12, 10, 11]		)







	def      A	(						self			:     Any		)			->		int:
								if not self.test_rust_tokenizer:
															return

								UpperCAmelCase :       Optional[int] 							=		self.get_tokenizer()
								UpperCAmelCase :       Any 							=		self.get_rust_tokenizer()

								UpperCAmelCase :       int 							=		'''UNwant\u00E9d,running'''

								UpperCAmelCase :       Dict 							=		tokenizer.tokenize(__snake_case		)
								UpperCAmelCase :       Optional[int] 							=		rust_tokenizer.tokenize(__snake_case		)
								self.assertListEqual(__snake_case		,			__snake_case		)

								UpperCAmelCase :       Optional[int] 							=		tokenizer.encode(__snake_case		,			add_special_tokens=__snake_case		)
								UpperCAmelCase :       int 							=		rust_tokenizer.encode(__snake_case		,			add_special_tokens=__snake_case		)
								self.assertListEqual(__snake_case		,			__snake_case		)

								UpperCAmelCase :       int 							=		self.get_rust_tokenizer()
								UpperCAmelCase :       int 							=		tokenizer.encode(__snake_case		)
								UpperCAmelCase :       str 							=		rust_tokenizer.encode(__snake_case		)
								self.assertListEqual(__snake_case		,			__snake_case		)

								# With lower casing
								UpperCAmelCase :       Dict 							=		self.get_tokenizer(do_lower_case=__snake_case		)
								UpperCAmelCase :       int 							=		self.get_rust_tokenizer(do_lower_case=__snake_case		)

								UpperCAmelCase :       Dict 							=		'''UNwant\u00E9d,running'''

								UpperCAmelCase :       int 							=		tokenizer.tokenize(__snake_case		)
								UpperCAmelCase :       List[Any] 							=		rust_tokenizer.tokenize(__snake_case		)
								self.assertListEqual(__snake_case		,			__snake_case		)

								UpperCAmelCase :       int 							=		tokenizer.encode(__snake_case		,			add_special_tokens=__snake_case		)
								UpperCAmelCase :       List[Any] 							=		rust_tokenizer.encode(__snake_case		,			add_special_tokens=__snake_case		)
								self.assertListEqual(__snake_case		,			__snake_case		)

								UpperCAmelCase :       Optional[Any] 							=		self.get_rust_tokenizer()
								UpperCAmelCase :       Union[str, Any] 							=		tokenizer.encode(__snake_case		)
								UpperCAmelCase :       Union[str, Any] 							=		rust_tokenizer.encode(__snake_case		)
								self.assertListEqual(__snake_case		,			__snake_case		)







	def      A	(						self			:     List[str]		)			->		List[str]:
								UpperCAmelCase :       str 							=		BasicTokenizer()

								self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''		)		,			['''ah''', '''\u535A''', '''\u63A8''', '''zz''']		)







	def      A	(						self			:     Optional[int]		)			->		Any:
								UpperCAmelCase :       Tuple 							=		BasicTokenizer(do_lower_case=__snake_case		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''		)		,			['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']		)
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)		,			['''hello''']		)







	def      A	(						self			:     Optional[int]		)			->		int:
								UpperCAmelCase :       Optional[Any] 							=		BasicTokenizer(do_lower_case=__snake_case		,			strip_accents=__snake_case		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)		,			['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']		)
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)		,			['''h\u00E9llo''']		)







	def      A	(						self			:     Union[str, Any]		)			->		Optional[Any]:
								UpperCAmelCase :       Dict 							=		BasicTokenizer(do_lower_case=__snake_case		,			strip_accents=__snake_case		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)		,			['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']		)
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)		,			['''hello''']		)







	def      A	(						self			:     str		)			->		Optional[int]:
								UpperCAmelCase :       Any 							=		BasicTokenizer(do_lower_case=__snake_case		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)		,			['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']		)
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)		,			['''hello''']		)







	def      A	(						self			:     Union[str, Any]		)			->		Optional[int]:
								UpperCAmelCase :       Dict 							=		BasicTokenizer(do_lower_case=__snake_case		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''		)		,			['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']		)







	def      A	(						self			:     Tuple		)			->		Any:
								UpperCAmelCase :       Dict 							=		BasicTokenizer(do_lower_case=__snake_case		,			strip_accents=__snake_case		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)		,			['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']		)







	def      A	(						self			:     Optional[int]		)			->		Union[str, Any]:
								UpperCAmelCase :       List[str] 							=		BasicTokenizer(do_lower_case=__snake_case		,			strip_accents=__snake_case		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)		,			['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']		)







	def      A	(						self			:     Optional[Any]		)			->		Union[str, Any]:
								UpperCAmelCase :       Tuple 							=		BasicTokenizer(do_lower_case=__snake_case		,			never_split=['''[UNK]''']		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''		)		,			['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']		)







	def      A	(						self			:     List[Any]		)			->		Dict:
								UpperCAmelCase :       List[Any] 							=		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

								UpperCAmelCase :       Tuple 							=		{}
								for i, token in enumerate(__snake_case		):
															UpperCAmelCase :       List[str] 							=		i
								UpperCAmelCase :       str 							=		WordpieceTokenizer(vocab=__snake_case		,			unk_token='''[UNK]'''		)

								self.assertListEqual(tokenizer.tokenize(''''''		)		,			[]		)

								self.assertListEqual(tokenizer.tokenize('''unwanted running'''		)		,			['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']		)

								self.assertListEqual(tokenizer.tokenize('''unwantedX running'''		)		,			['''[UNK]''', '''runn''', '''##ing''']		)







	def      A	(						self			:     Union[str, Any]		)			->		Tuple:
								self.assertTrue(_is_whitespace(''' '''		)		)
								self.assertTrue(_is_whitespace('''\t'''		)		)
								self.assertTrue(_is_whitespace('''\r'''		)		)
								self.assertTrue(_is_whitespace('''\n'''		)		)
								self.assertTrue(_is_whitespace('''\u00A0'''		)		)

								self.assertFalse(_is_whitespace('''A'''		)		)
								self.assertFalse(_is_whitespace('''-'''		)		)







	def      A	(						self			:     Union[str, Any]		)			->		Any:
								self.assertTrue(_is_control('''\u0005'''		)		)

								self.assertFalse(_is_control('''A'''		)		)
								self.assertFalse(_is_control(''' '''		)		)
								self.assertFalse(_is_control('''\t'''		)		)
								self.assertFalse(_is_control('''\r'''		)		)







	def      A	(						self			:     Optional[int]		)			->		Tuple:
								self.assertTrue(_is_punctuation('''-'''		)		)
								self.assertTrue(_is_punctuation('''$'''		)		)
								self.assertTrue(_is_punctuation('''`'''		)		)
								self.assertTrue(_is_punctuation('''.'''		)		)

								self.assertFalse(_is_punctuation('''A'''		)		)
								self.assertFalse(_is_punctuation(''' '''		)		)







	def      A	(						self			:     Union[str, Any]		)			->		List[str]:
								UpperCAmelCase :       Dict 							=		self.get_tokenizer()
								UpperCAmelCase :       int 							=		self.get_rust_tokenizer()

								# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
								self.assertListEqual([tokenizer.tokenize(__snake_case		) for t in ['''Test''', '''\xad''', '''test''']]		,			[['''[UNK]'''], [], ['''[UNK]''']]		)

								self.assertListEqual(
								    [rust_tokenizer.tokenize(__snake_case		) for t in ['''Test''', '''\xad''', '''test''']]		,			[['''[UNK]'''], [], ['''[UNK]''']]		)







	@slow
	def      A	(						self			:     Union[str, Any]		)			->		Optional[int]:
								UpperCAmelCase :       Optional[Any] 							=		self.tokenizer_class.from_pretrained('''google/mobilebert-uncased'''		)

								UpperCAmelCase :       Dict 							=		tokenizer.encode('''sequence builders'''		,			add_special_tokens=__snake_case		)
								UpperCAmelCase :       List[str] 							=		tokenizer.encode('''multi-sequence build'''		,			add_special_tokens=__snake_case		)

								UpperCAmelCase :       Any 							=		tokenizer.build_inputs_with_special_tokens(__snake_case		)
								UpperCAmelCase :       Tuple 							=		tokenizer.build_inputs_with_special_tokens(__snake_case		,			__snake_case		)

								assert encoded_sentence == [101] + text + [102]
								assert encoded_pair == [101] + text + [102] + text_a + [102]







	def      A	(						self			:     Optional[Any]		)			->		Any:
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
																						UpperCAmelCase :       Dict 							=		self.rust_tokenizer_class.from_pretrained(__snake_case		,			**__snake_case		)

																						UpperCAmelCase :       str 							=		F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																						UpperCAmelCase :       Optional[int] 							=		tokenizer_r.encode_plus(
																						    __snake_case		,			return_attention_mask=__snake_case		,			return_token_type_ids=__snake_case		,			return_offsets_mapping=__snake_case		,			add_special_tokens=__snake_case		,			)

																						UpperCAmelCase :       List[Any] 							=		tokenizer_r.do_lower_case if hasattr(__snake_case		,			'''do_lower_case'''		) else False
																						UpperCAmelCase :       str 							=		(
																						    [
																						        ((0, 0), tokenizer_r.cls_token),
																						        ((0, 1), '''A'''),
																						        ((1, 2), ''','''),
																						        ((3, 5), '''na'''),
																						        ((5, 6), '''##ï'''),
																						        ((6, 8), '''##ve'''),
																						        ((9, 15), tokenizer_r.mask_token),
																						        ((16, 21), '''Allen'''),
																						        ((21, 23), '''##NL'''),
																						        ((23, 24), '''##P'''),
																						        ((25, 33), '''sentence'''),
																						        ((33, 34), '''.'''),
																						        ((0, 0), tokenizer_r.sep_token),
																						    ]
																						    if not do_lower_case
																						    else [
																						        ((0, 0), tokenizer_r.cls_token),
																						        ((0, 1), '''a'''),
																						        ((1, 2), ''','''),
																						        ((3, 8), '''naive'''),
																						        ((9, 15), tokenizer_r.mask_token),
																						        ((16, 21), '''allen'''),
																						        ((21, 23), '''##nl'''),
																						        ((23, 24), '''##p'''),
																						        ((25, 33), '''sentence'''),
																						        ((33, 34), '''.'''),
																						        ((0, 0), tokenizer_r.sep_token),
																						    ]
																						)

																						self.assertEqual(
																						    [e[1] for e in expected_results]		,			tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']		)		)
																						self.assertEqual([e[0] for e in expected_results]		,			tokens['''offset_mapping''']		)







	def      A	(						self			:     Optional[int]		)			->		str:
								UpperCAmelCase :       str 							=		['''的''', '''人''', '''有''']
								UpperCAmelCase :       List[Any] 							=		''''''.join(__snake_case		)
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
																						UpperCAmelCase :       Union[str, Any] 							=		True
																						UpperCAmelCase :       List[str] 							=		self.tokenizer_class.from_pretrained(__snake_case		,			**__snake_case		)
																						UpperCAmelCase :       int 							=		self.rust_tokenizer_class.from_pretrained(__snake_case		,			**__snake_case		)

																						UpperCAmelCase :       List[Any] 							=		tokenizer_p.encode(__snake_case		,			add_special_tokens=__snake_case		)
																						UpperCAmelCase :       Tuple 							=		tokenizer_r.encode(__snake_case		,			add_special_tokens=__snake_case		)

																						UpperCAmelCase :       Any 							=		tokenizer_r.convert_ids_to_tokens(__snake_case		)
																						UpperCAmelCase :       Optional[int] 							=		tokenizer_p.convert_ids_to_tokens(__snake_case		)

																						# it is expected that each Chinese character is not preceded by "##"
																						self.assertListEqual(__snake_case		,			__snake_case		)
																						self.assertListEqual(__snake_case		,			__snake_case		)

																						UpperCAmelCase :       Union[str, Any] 							=		False
																						UpperCAmelCase :       int 							=		self.rust_tokenizer_class.from_pretrained(__snake_case		,			**__snake_case		)
																						UpperCAmelCase :       int 							=		self.tokenizer_class.from_pretrained(__snake_case		,			**__snake_case		)

																						UpperCAmelCase :       List[Any] 							=		tokenizer_r.encode(__snake_case		,			add_special_tokens=__snake_case		)
																						UpperCAmelCase :       Tuple 							=		tokenizer_p.encode(__snake_case		,			add_special_tokens=__snake_case		)

																						UpperCAmelCase :       List[str] 							=		tokenizer_r.convert_ids_to_tokens(__snake_case		)
																						UpperCAmelCase :       Optional[int] 							=		tokenizer_p.convert_ids_to_tokens(__snake_case		)

																						# it is expected that only the first Chinese character is not preceded by "##".
																						UpperCAmelCase :       Optional[Any] 							=		[
																						    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(__snake_case		)
																						]
																						self.assertListEqual(__snake_case		,			__snake_case		)
																						self.assertListEqual(__snake_case		,			__snake_case		)









'''simple docstring'''

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase							:   int  						=	logging.get_logger(__name__)

lowerCamelCase							:   List[Any]  						=	{
    'google/realm-cc-news-pretrained-embedder': (
        'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/config.json'
    ),
    'google/realm-cc-news-pretrained-encoder': (
        'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/config.json'
    ),
    'google/realm-cc-news-pretrained-scorer': (
        'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/config.json'
    ),
    'google/realm-cc-news-pretrained-openqa': (
        'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/config.json'
    ),
    'google/realm-orqa-nq-openqa': 'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/config.json',
    'google/realm-orqa-nq-reader': 'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/config.json',
    'google/realm-orqa-wq-openqa': 'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/config.json',
    'google/realm-orqa-wq-reader': 'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/config.json',
    # See all REALM models at https://huggingface.co/models?filter=realm
}
class   __lowerCAmelCase						(lowercase_     ):





      '''simple docstring'''






      lowerCAmelCase__					:     Optional[int]  					=							"""realm"""



      def __init__(self    :    str   ,						UpperCamelCase    :    List[Any]=30522   ,						UpperCamelCase    :    List[Any]=768   ,						UpperCamelCase    :    int=128   ,						UpperCamelCase    :    Any=12   ,						UpperCamelCase    :    Tuple=12   ,						UpperCamelCase    :    List[Any]=8   ,						UpperCamelCase    :    Union[str, Any]=3072   ,						UpperCamelCase    :    List[str]="gelu_new"   ,						UpperCamelCase    :    Any=0.1   ,						UpperCamelCase    :    List[str]=0.1   ,						UpperCamelCase    :    Dict=512   ,						UpperCamelCase    :    Dict=2   ,						UpperCamelCase    :    List[Any]=0.02   ,						UpperCamelCase    :    List[Any]=1E-12   ,						UpperCamelCase    :    Dict=256   ,						UpperCamelCase    :    Union[str, Any]=10   ,						UpperCamelCase    :    Optional[int]=1E-3   ,						UpperCamelCase    :    Tuple=5   ,						UpperCamelCase    :    Optional[int]=320   ,						UpperCamelCase    :    List[str]=13353718   ,						UpperCamelCase    :    Optional[Any]=5000   ,						UpperCamelCase    :    str=1   ,						UpperCamelCase    :    Union[str, Any]=0   ,						UpperCamelCase    :    List[Any]=2   ,						**UpperCamelCase    :    int   ,						):

             '''simple docstring'''







             super().__init__(pad_token_id=UpperCamelCase   ,						bos_token_id=UpperCamelCase   ,						eos_token_id=UpperCamelCase   ,						**UpperCamelCase )

             # Common config
             lowercase__      			= vocab_size
             lowercase__      			= max_position_embeddings
             lowercase__      			= hidden_size
             lowercase__      			= retriever_proj_size
             lowercase__      			= num_hidden_layers
             lowercase__      			= num_attention_heads
             lowercase__      			= num_candidates
             lowercase__      			= intermediate_size
             lowercase__      			= hidden_act
             lowercase__      			= hidden_dropout_prob
             lowercase__      			= attention_probs_dropout_prob
             lowercase__      			= initializer_range
             lowercase__      			= type_vocab_size
             lowercase__      			= layer_norm_eps

             # Reader config
             lowercase__      			= span_hidden_size
             lowercase__      			= max_span_width
             lowercase__      			= reader_layer_norm_eps
             lowercase__      			= reader_beam_size
             lowercase__      			= reader_seq_len

             # Retrieval config
             lowercase__      			= num_block_records
             lowercase__      			= searcher_beam_size




