



'''simple docstring'''






import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 				SCREAMING_SNAKE_CASE(							A__							,		unittest.TestCase			):







	"""simple docstring"""






	lowerCamelCase__			 =       MobileBertTokenizer
	lowerCamelCase__			 =       MobileBertTokenizerFast
	lowerCamelCase__			 =       True
	lowerCamelCase__			 =       True
	lowerCamelCase__			 =       filter_non_english
	lowerCamelCase__			 =       """google/mobilebert-uncased"""







	def      A	(						self			:     Any		)			->		int:
								super().setUp()

								UpperCAmelCase :       str 							=		[
								    '''[UNK]''',
								    '''[CLS]''',
								    '''[SEP]''',
								    '''[PAD]''',
								    '''[MASK]''',
								    '''want''',
								    '''##want''',
								    '''##ed''',
								    '''wa''',
								    '''un''',
								    '''runn''',
								    '''##ing''',
								    ''',''',
								    '''low''',
								    '''lowest''',
								]
								UpperCAmelCase :       Dict 							=		os.path.join(self.tmpdirname		,			VOCAB_FILES_NAMES['''vocab_file''']		)
								with open(self.vocab_file		,			'''w'''		,			encoding='''utf-8'''		) as vocab_writer:
															vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]		)		)

								UpperCAmelCase :       List[Any] 							=		[
								    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
								    for tokenizer_def in self.tokenizers_list
								]







	def      A	(						self			:     Optional[Any]		,			__snake_case			:     Union[str, Any]		)			->		Optional[Any]:
								UpperCAmelCase :       Optional[Any] 							=		'''UNwant\u00E9d,running'''
								UpperCAmelCase :       int 							=		'''unwanted, running'''
								return input_text, output_text







	def      A	(						self			:     Optional[Any]		)			->		Dict:
								UpperCAmelCase :       List[str] 							=		self.tokenizer_class(self.vocab_file		)

								UpperCAmelCase :       Dict 							=		tokenizer.tokenize('''UNwant\u00E9d,running'''		)
								self.assertListEqual(__snake_case		,			['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']		)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case		)		,			[9, 6, 7, 12, 10, 11]		)







	def      A	(						self			:     Any		)			->		int:
								if not self.test_rust_tokenizer:
															return

								UpperCAmelCase :       Optional[int] 							=		self.get_tokenizer()
								UpperCAmelCase :       Any 							=		self.get_rust_tokenizer()

								UpperCAmelCase :       int 							=		'''UNwant\u00E9d,running'''

								UpperCAmelCase :       Dict 							=		tokenizer.tokenize(__snake_case		)
								UpperCAmelCase :       Optional[int] 							=		rust_tokenizer.tokenize(__snake_case		)
								self.assertListEqual(__snake_case		,			__snake_case		)

								UpperCAmelCase :       Optional[int] 							=		tokenizer.encode(__snake_case		,			add_special_tokens=__snake_case		)
								UpperCAmelCase :       int 							=		rust_tokenizer.encode(__snake_case		,			add_special_tokens=__snake_case		)
								self.assertListEqual(__snake_case		,			__snake_case		)

								UpperCAmelCase :       int 							=		self.get_rust_tokenizer()
								UpperCAmelCase :       int 							=		tokenizer.encode(__snake_case		)
								UpperCAmelCase :       str 							=		rust_tokenizer.encode(__snake_case		)
								self.assertListEqual(__snake_case		,			__snake_case		)

								# With lower casing
								UpperCAmelCase :       Dict 							=		self.get_tokenizer(do_lower_case=__snake_case		)
								UpperCAmelCase :       int 							=		self.get_rust_tokenizer(do_lower_case=__snake_case		)

								UpperCAmelCase :       Dict 							=		'''UNwant\u00E9d,running'''

								UpperCAmelCase :       int 							=		tokenizer.tokenize(__snake_case		)
								UpperCAmelCase :       List[Any] 							=		rust_tokenizer.tokenize(__snake_case		)
								self.assertListEqual(__snake_case		,			__snake_case		)

								UpperCAmelCase :       int 							=		tokenizer.encode(__snake_case		,			add_special_tokens=__snake_case		)
								UpperCAmelCase :       List[Any] 							=		rust_tokenizer.encode(__snake_case		,			add_special_tokens=__snake_case		)
								self.assertListEqual(__snake_case		,			__snake_case		)

								UpperCAmelCase :       Optional[Any] 							=		self.get_rust_tokenizer()
								UpperCAmelCase :       Union[str, Any] 							=		tokenizer.encode(__snake_case		)
								UpperCAmelCase :       Union[str, Any] 							=		rust_tokenizer.encode(__snake_case		)
								self.assertListEqual(__snake_case		,			__snake_case		)







	def      A	(						self			:     List[str]		)			->		List[str]:
								UpperCAmelCase :       str 							=		BasicTokenizer()

								self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''		)		,			['''ah''', '''\u535A''', '''\u63A8''', '''zz''']		)







	def      A	(						self			:     Optional[int]		)			->		Any:
								UpperCAmelCase :       Tuple 							=		BasicTokenizer(do_lower_case=__snake_case		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''		)		,			['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']		)
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)		,			['''hello''']		)







	def      A	(						self			:     Optional[int]		)			->		int:
								UpperCAmelCase :       Optional[Any] 							=		BasicTokenizer(do_lower_case=__snake_case		,			strip_accents=__snake_case		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)		,			['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']		)
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)		,			['''h\u00E9llo''']		)







	def      A	(						self			:     Union[str, Any]		)			->		Optional[Any]:
								UpperCAmelCase :       Dict 							=		BasicTokenizer(do_lower_case=__snake_case		,			strip_accents=__snake_case		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)		,			['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']		)
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)		,			['''hello''']		)







	def      A	(						self			:     str		)			->		Optional[int]:
								UpperCAmelCase :       Any 							=		BasicTokenizer(do_lower_case=__snake_case		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)		,			['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']		)
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)		,			['''hello''']		)







	def      A	(						self			:     Union[str, Any]		)			->		Optional[int]:
								UpperCAmelCase :       Dict 							=		BasicTokenizer(do_lower_case=__snake_case		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''		)		,			['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']		)







	def      A	(						self			:     Tuple		)			->		Any:
								UpperCAmelCase :       Dict 							=		BasicTokenizer(do_lower_case=__snake_case		,			strip_accents=__snake_case		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)		,			['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']		)







	def      A	(						self			:     Optional[int]		)			->		Union[str, Any]:
								UpperCAmelCase :       List[str] 							=		BasicTokenizer(do_lower_case=__snake_case		,			strip_accents=__snake_case		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)		,			['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']		)







	def      A	(						self			:     Optional[Any]		)			->		Union[str, Any]:
								UpperCAmelCase :       Tuple 							=		BasicTokenizer(do_lower_case=__snake_case		,			never_split=['''[UNK]''']		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''		)		,			['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']		)







	def      A	(						self			:     List[Any]		)			->		Dict:
								UpperCAmelCase :       List[Any] 							=		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

								UpperCAmelCase :       Tuple 							=		{}
								for i, token in enumerate(__snake_case		):
															UpperCAmelCase :       List[str] 							=		i
								UpperCAmelCase :       str 							=		WordpieceTokenizer(vocab=__snake_case		,			unk_token='''[UNK]'''		)

								self.assertListEqual(tokenizer.tokenize(''''''		)		,			[]		)

								self.assertListEqual(tokenizer.tokenize('''unwanted running'''		)		,			['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']		)

								self.assertListEqual(tokenizer.tokenize('''unwantedX running'''		)		,			['''[UNK]''', '''runn''', '''##ing''']		)







	def      A	(						self			:     Union[str, Any]		)			->		Tuple:
								self.assertTrue(_is_whitespace(''' '''		)		)
								self.assertTrue(_is_whitespace('''\t'''		)		)
								self.assertTrue(_is_whitespace('''\r'''		)		)
								self.assertTrue(_is_whitespace('''\n'''		)		)
								self.assertTrue(_is_whitespace('''\u00A0'''		)		)

								self.assertFalse(_is_whitespace('''A'''		)		)
								self.assertFalse(_is_whitespace('''-'''		)		)







	def      A	(						self			:     Union[str, Any]		)			->		Any:
								self.assertTrue(_is_control('''\u0005'''		)		)

								self.assertFalse(_is_control('''A'''		)		)
								self.assertFalse(_is_control(''' '''		)		)
								self.assertFalse(_is_control('''\t'''		)		)
								self.assertFalse(_is_control('''\r'''		)		)







	def      A	(						self			:     Optional[int]		)			->		Tuple:
								self.assertTrue(_is_punctuation('''-'''		)		)
								self.assertTrue(_is_punctuation('''$'''		)		)
								self.assertTrue(_is_punctuation('''`'''		)		)
								self.assertTrue(_is_punctuation('''.'''		)		)

								self.assertFalse(_is_punctuation('''A'''		)		)
								self.assertFalse(_is_punctuation(''' '''		)		)







	def      A	(						self			:     Union[str, Any]		)			->		List[str]:
								UpperCAmelCase :       Dict 							=		self.get_tokenizer()
								UpperCAmelCase :       int 							=		self.get_rust_tokenizer()

								# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
								self.assertListEqual([tokenizer.tokenize(__snake_case		) for t in ['''Test''', '''\xad''', '''test''']]		,			[['''[UNK]'''], [], ['''[UNK]''']]		)

								self.assertListEqual(
								    [rust_tokenizer.tokenize(__snake_case		) for t in ['''Test''', '''\xad''', '''test''']]		,			[['''[UNK]'''], [], ['''[UNK]''']]		)







	@slow
	def      A	(						self			:     Union[str, Any]		)			->		Optional[int]:
								UpperCAmelCase :       Optional[Any] 							=		self.tokenizer_class.from_pretrained('''google/mobilebert-uncased'''		)

								UpperCAmelCase :       Dict 							=		tokenizer.encode('''sequence builders'''		,			add_special_tokens=__snake_case		)
								UpperCAmelCase :       List[str] 							=		tokenizer.encode('''multi-sequence build'''		,			add_special_tokens=__snake_case		)

								UpperCAmelCase :       Any 							=		tokenizer.build_inputs_with_special_tokens(__snake_case		)
								UpperCAmelCase :       Tuple 							=		tokenizer.build_inputs_with_special_tokens(__snake_case		,			__snake_case		)

								assert encoded_sentence == [101] + text + [102]
								assert encoded_pair == [101] + text + [102] + text_a + [102]







	def      A	(						self			:     Optional[Any]		)			->		Any:
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
																						UpperCAmelCase :       Dict 							=		self.rust_tokenizer_class.from_pretrained(__snake_case		,			**__snake_case		)

																						UpperCAmelCase :       str 							=		F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																						UpperCAmelCase :       Optional[int] 							=		tokenizer_r.encode_plus(
																						    __snake_case		,			return_attention_mask=__snake_case		,			return_token_type_ids=__snake_case		,			return_offsets_mapping=__snake_case		,			add_special_tokens=__snake_case		,			)

																						UpperCAmelCase :       List[Any] 							=		tokenizer_r.do_lower_case if hasattr(__snake_case		,			'''do_lower_case'''		) else False
																						UpperCAmelCase :       str 							=		(
																						    [
																						        ((0, 0), tokenizer_r.cls_token),
																						        ((0, 1), '''A'''),
																						        ((1, 2), ''','''),
																						        ((3, 5), '''na'''),
																						        ((5, 6), '''##ï'''),
																						        ((6, 8), '''##ve'''),
																						        ((9, 15), tokenizer_r.mask_token),
																						        ((16, 21), '''Allen'''),
																						        ((21, 23), '''##NL'''),
																						        ((23, 24), '''##P'''),
																						        ((25, 33), '''sentence'''),
																						        ((33, 34), '''.'''),
																						        ((0, 0), tokenizer_r.sep_token),
																						    ]
																						    if not do_lower_case
																						    else [
																						        ((0, 0), tokenizer_r.cls_token),
																						        ((0, 1), '''a'''),
																						        ((1, 2), ''','''),
																						        ((3, 8), '''naive'''),
																						        ((9, 15), tokenizer_r.mask_token),
																						        ((16, 21), '''allen'''),
																						        ((21, 23), '''##nl'''),
																						        ((23, 24), '''##p'''),
																						        ((25, 33), '''sentence'''),
																						        ((33, 34), '''.'''),
																						        ((0, 0), tokenizer_r.sep_token),
																						    ]
																						)

																						self.assertEqual(
																						    [e[1] for e in expected_results]		,			tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']		)		)
																						self.assertEqual([e[0] for e in expected_results]		,			tokens['''offset_mapping''']		)







	def      A	(						self			:     Optional[int]		)			->		str:
								UpperCAmelCase :       str 							=		['''的''', '''人''', '''有''']
								UpperCAmelCase :       List[Any] 							=		''''''.join(__snake_case		)
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
																						UpperCAmelCase :       Union[str, Any] 							=		True
																						UpperCAmelCase :       List[str] 							=		self.tokenizer_class.from_pretrained(__snake_case		,			**__snake_case		)
																						UpperCAmelCase :       int 							=		self.rust_tokenizer_class.from_pretrained(__snake_case		,			**__snake_case		)

																						UpperCAmelCase :       List[Any] 							=		tokenizer_p.encode(__snake_case		,			add_special_tokens=__snake_case		)
																						UpperCAmelCase :       Tuple 							=		tokenizer_r.encode(__snake_case		,			add_special_tokens=__snake_case		)

																						UpperCAmelCase :       Any 							=		tokenizer_r.convert_ids_to_tokens(__snake_case		)
																						UpperCAmelCase :       Optional[int] 							=		tokenizer_p.convert_ids_to_tokens(__snake_case		)

																						# it is expected that each Chinese character is not preceded by "##"
																						self.assertListEqual(__snake_case		,			__snake_case		)
																						self.assertListEqual(__snake_case		,			__snake_case		)

																						UpperCAmelCase :       Union[str, Any] 							=		False
																						UpperCAmelCase :       int 							=		self.rust_tokenizer_class.from_pretrained(__snake_case		,			**__snake_case		)
																						UpperCAmelCase :       int 							=		self.tokenizer_class.from_pretrained(__snake_case		,			**__snake_case		)

																						UpperCAmelCase :       List[Any] 							=		tokenizer_r.encode(__snake_case		,			add_special_tokens=__snake_case		)
																						UpperCAmelCase :       Tuple 							=		tokenizer_p.encode(__snake_case		,			add_special_tokens=__snake_case		)

																						UpperCAmelCase :       List[str] 							=		tokenizer_r.convert_ids_to_tokens(__snake_case		)
																						UpperCAmelCase :       Optional[int] 							=		tokenizer_p.convert_ids_to_tokens(__snake_case		)

																						# it is expected that only the first Chinese character is not preceded by "##".
																						UpperCAmelCase :       Optional[Any] 							=		[
																						    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(__snake_case		)
																						]
																						self.assertListEqual(__snake_case		,			__snake_case		)
																						self.assertListEqual(__snake_case		,			__snake_case		)









'''simple docstring'''




from ...configuration_utils import PretrainedConfig
from ...utils import logging


__snake_case    						=logging.get_logger(__name__)

__snake_case    						={
    """facebook/s2t-wav2vec2-large-en-de""": (
        """https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/config.json"""
    ),
    # See all Speech2Text models at https://huggingface.co/models?filter=speech2text2
}

class 		UpperCAmelCase_      (  __lowercase		):
     lowerCamelCase				:	Union[str, Any]		=     '''speech_to_text_2'''
     lowerCamelCase				:	Any		=     ['''past_key_values''']
     lowerCamelCase				:	Optional[Any]		=     {'''num_attention_heads''': '''decoder_attention_heads''', '''hidden_size''': '''d_model'''}





     def __init__( self       :      Optional[int]    ,      UpperCAmelCase__       :      Optional[Any]=1_0_0_0_0    ,      UpperCAmelCase__       :      int=6    ,      UpperCAmelCase__       :      Optional[Any]=2_0_4_8    ,      UpperCAmelCase__       :      Union[str, Any]=4    ,      UpperCAmelCase__       :      List[Any]=0.0    ,      UpperCAmelCase__       :      str=True    ,      UpperCAmelCase__       :      str="relu"    ,      UpperCAmelCase__       :      Any=2_5_6    ,      UpperCAmelCase__       :      Optional[Any]=0.1    ,      UpperCAmelCase__       :      Any=0.0    ,      UpperCAmelCase__       :      Tuple=0.0    ,      UpperCAmelCase__       :      List[Any]=0.02    ,      UpperCAmelCase__       :      List[Any]=2    ,      UpperCAmelCase__       :      str=True    ,      UpperCAmelCase__       :      List[str]=1    ,      UpperCAmelCase__       :      Any=0    ,      UpperCAmelCase__       :      Dict=2    ,      UpperCAmelCase__       :      int=1_0_2_4    ,      **UpperCAmelCase__       :      Optional[Any]    ,      )				->  Dict:
        lowerCAmelCase     =			vocab_size
        lowerCAmelCase     =			d_model
        lowerCAmelCase     =			decoder_ffn_dim
        lowerCAmelCase     =			decoder_layers
        lowerCAmelCase     =			decoder_attention_heads
        lowerCAmelCase     =			dropout
        lowerCAmelCase     =			attention_dropout
        lowerCAmelCase     =			activation_dropout
        lowerCAmelCase     =			activation_function
        lowerCAmelCase     =			init_std
        lowerCAmelCase     =			decoder_layerdrop
        lowerCAmelCase     =			use_cache
        lowerCAmelCase     =			decoder_layers
        lowerCAmelCase     =			scale_embedding  # scale factor will be sqrt(d_model) if True
        lowerCAmelCase     =			max_target_positions

        super().__init__(
            pad_token_id=UpperCAmelCase__    ,      bos_token_id=UpperCAmelCase__    ,      eos_token_id=UpperCAmelCase__    ,      decoder_start_token_id=UpperCAmelCase__    ,      **UpperCAmelCase__    ,      )





