




'''simple docstring'''

import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin






class 						a				(   _lowerCamelCase   ,     unittest.TestCase     ):
				snake_case_      =		ProphetNetTokenizer
				snake_case_      =		False


				def   A_ (		self    :					Optional[Any]					):
									super().setUp()

									snake_case_									=				[
									    '''[UNK]''',
									    '''[CLS]''',
									    '''[SEP]''',
									    '''[PAD]''',
									    '''[MASK]''',
									    '''want''',
									    '''##want''',
									    '''##ed''',
									    '''wa''',
									    '''un''',
									    '''runn''',
									    '''##ing''',
									    ''',''',
									    '''low''',
									    '''lowest''',
									]
									snake_case_									=				os.path.join(self.tmpdirname       ,				VOCAB_FILES_NAMES['''vocab_file''']					)
									with open(self.vocab_file       ,				'''w'''       ,				encoding='''utf-8'''					) as vocab_writer:
														vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]					)					)


				def   A_ (		self    :					List[Any]       ,				lowercase_    :					Tuple					):
									snake_case_									=				'''UNwant\u00E9d,running'''
									snake_case_									=				'''unwanted, running'''
									return input_text, output_text


				def   A_ (		self    :					Any					):
									snake_case_									=				self.tokenizer_class(self.vocab_file					)

									snake_case_									=				tokenizer.tokenize('''UNwant\u00E9d,running'''					)
									self.assertListEqual(lowercase_       ,				['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']					)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_					)       ,				[9, 6, 7, 12, 10, 11]					)


				def   A_ (		self    :					str					):
									snake_case_									=				BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''					)       ,				['''ah''', '''\u535A''', '''\u63A8''', '''zz''']					)


				def   A_ (		self    :					str					):
									snake_case_									=				BasicTokenizer(do_lower_case=lowercase_					)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)       ,				['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)       ,				['''hello''']					)


				def   A_ (		self    :					Dict					):
									snake_case_									=				BasicTokenizer(do_lower_case=lowercase_       ,				strip_accents=lowercase_					)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)       ,				['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)       ,				['''h\u00E9llo''']					)


				def   A_ (		self    :					Any					):
									snake_case_									=				BasicTokenizer(do_lower_case=lowercase_       ,				strip_accents=lowercase_					)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)       ,				['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)       ,				['''hello''']					)


				def   A_ (		self    :					str					):
									snake_case_									=				BasicTokenizer(do_lower_case=lowercase_					)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)       ,				['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)       ,				['''hello''']					)


				def   A_ (		self    :					List[Any]					):
									snake_case_									=				BasicTokenizer(do_lower_case=lowercase_					)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)       ,				['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)


				def   A_ (		self    :					Any					):
									snake_case_									=				BasicTokenizer(do_lower_case=lowercase_       ,				strip_accents=lowercase_					)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)       ,				['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)


				def   A_ (		self    :					Union[str, Any]					):
									snake_case_									=				BasicTokenizer(do_lower_case=lowercase_       ,				strip_accents=lowercase_					)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)       ,				['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)


				def   A_ (		self    :					Any					):
									snake_case_									=				BasicTokenizer(do_lower_case=lowercase_       ,				never_split=['''[UNK]''']					)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''					)       ,				['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']					)


				def   A_ (		self    :					Dict					):
									snake_case_									=				['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

									snake_case_									=				{}
									for i, token in enumerate(lowercase_					):
														snake_case_									=				i
									snake_case_									=				WordpieceTokenizer(vocab=lowercase_       ,				unk_token='''[UNK]'''					)

									self.assertListEqual(tokenizer.tokenize(''''''					)       ,				[]					)

									self.assertListEqual(tokenizer.tokenize('''unwanted running'''					)       ,				['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']					)

									self.assertListEqual(tokenizer.tokenize('''unwantedX running'''					)       ,				['''[UNK]''', '''runn''', '''##ing''']					)


				@require_torch
				def   A_ (		self    :					List[Any]					):
									snake_case_									=				self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased'''					)

									snake_case_									=				['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
									snake_case_									=				[1037, 2146, 2_0423, 2005, 7680, 7849, 3989, 1012, 102]
									snake_case_									=				tokenizer(lowercase_       ,				padding=lowercase_       ,				return_tensors='''pt'''					)
									self.assertIsInstance(lowercase_       ,				lowercase_					)
									snake_case_									=				list(batch.input_ids.numpy()[0]					)
									self.assertListEqual(lowercase_       ,				lowercase_					)

									self.assertEqual((2, 9)       ,				batch.input_ids.shape					)
									self.assertEqual((2, 9)       ,				batch.attention_mask.shape					)


				def   A_ (		self    :					Tuple					):
									self.assertTrue(_is_whitespace(''' '''					)					)
									self.assertTrue(_is_whitespace('''\t'''					)					)
									self.assertTrue(_is_whitespace('''\r'''					)					)
									self.assertTrue(_is_whitespace('''\n'''					)					)
									self.assertTrue(_is_whitespace('''\u00A0'''					)					)

									self.assertFalse(_is_whitespace('''A'''					)					)
									self.assertFalse(_is_whitespace('''-'''					)					)


				def   A_ (		self    :					int					):
									self.assertTrue(_is_control('''\u0005'''					)					)

									self.assertFalse(_is_control('''A'''					)					)
									self.assertFalse(_is_control(''' '''					)					)
									self.assertFalse(_is_control('''\t'''					)					)
									self.assertFalse(_is_control('''\r'''					)					)


				def   A_ (		self    :					Optional[int]					):
									self.assertTrue(_is_punctuation('''-'''					)					)
									self.assertTrue(_is_punctuation('''$'''					)					)
									self.assertTrue(_is_punctuation('''`'''					)					)
									self.assertTrue(_is_punctuation('''.'''					)					)

									self.assertFalse(_is_punctuation('''A'''					)					)
									self.assertFalse(_is_punctuation(''' '''					)					)


				@slow
				def   A_ (		self    :					Optional[Any]					):
									snake_case_									=				self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased'''					)

									snake_case_									=				tokenizer.encode('''sequence builders'''       ,				add_special_tokens=lowercase_					)
									snake_case_									=				tokenizer.encode('''multi-sequence build'''       ,				add_special_tokens=lowercase_					)

									snake_case_									=				tokenizer.build_inputs_with_special_tokens(lowercase_					)
									snake_case_									=				tokenizer.build_inputs_with_special_tokens(lowercase_       ,				lowercase_					)

									assert encoded_sentence == text + [102]
									assert encoded_pair == text + [102] + text_a + [102]







'''simple docstring'''

import unittest

from transformers import MPNetConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MPNetForMaskedLM,
							    MPNetForMultipleChoice,
							    MPNetForQuestionAnswering,
							    MPNetForSequenceClassification,
							    MPNetForTokenClassification,
							    MPNetModel,
							)






class 						a				:


				def __init__(		self    :					List[Any]       ,				lowercase_    :					Optional[Any]       ,				lowercase_    :					Any=13       ,				lowercase_    :					Optional[Any]=7       ,				lowercase_    :					Optional[Any]=True       ,				lowercase_    :					Dict=True       ,				lowercase_    :					Tuple=False       ,				lowercase_    :					Optional[Any]=True       ,				lowercase_    :					Any=99       ,				lowercase_    :					Union[str, Any]=64       ,				lowercase_    :					str=5       ,				lowercase_    :					int=4       ,				lowercase_    :					List[Any]=64       ,				lowercase_    :					Dict="gelu"       ,				lowercase_    :					Optional[int]=0.1       ,				lowercase_    :					Optional[Any]=0.1       ,				lowercase_    :					Tuple=512       ,				lowercase_    :					List[Any]=16       ,				lowercase_    :					str=2       ,				lowercase_    :					List[str]=0.02       ,				lowercase_    :					Optional[Any]=3       ,				lowercase_    :					Optional[Any]=4       ,				lowercase_    :					List[Any]=None       ,				):
									snake_case_									=				parent
									snake_case_									=				batch_size
									snake_case_									=				seq_length
									snake_case_									=				is_training
									snake_case_									=				use_input_mask
									snake_case_									=				use_token_type_ids
									snake_case_									=				use_labels
									snake_case_									=				vocab_size
									snake_case_									=				hidden_size
									snake_case_									=				num_hidden_layers
									snake_case_									=				num_attention_heads
									snake_case_									=				intermediate_size
									snake_case_									=				hidden_act
									snake_case_									=				hidden_dropout_prob
									snake_case_									=				attention_probs_dropout_prob
									snake_case_									=				max_position_embeddings
									snake_case_									=				type_vocab_size
									snake_case_									=				type_sequence_label_size
									snake_case_									=				initializer_range
									snake_case_									=				num_labels
									snake_case_									=				num_choices
									snake_case_									=				scope


				def   A_ (		self    :					List[str]					):
									return MPNetConfig.from_pretrained('''microsoft/mpnet-base'''					)


				def   A_ (		self    :					str					):
									snake_case_									=				ids_tensor([self.batch_size, self.seq_length]       ,				self.vocab_size					)

									snake_case_									=				None
									if self.use_input_mask:
														snake_case_									=				random_attention_mask([self.batch_size, self.seq_length]					)

									snake_case_									=				None
									snake_case_									=				None
									snake_case_									=				None
									if self.use_labels:
														snake_case_									=				ids_tensor([self.batch_size]       ,				self.type_sequence_label_size					)
														snake_case_									=				ids_tensor([self.batch_size, self.seq_length]       ,				self.num_labels					)
														snake_case_									=				ids_tensor([self.batch_size]       ,				self.num_choices					)

									snake_case_									=				self.get_config()
									return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels


				def   A_ (		self    :					Tuple					):
									return MPNetConfig(
									    vocab_size=self.vocab_size       ,				hidden_size=self.hidden_size       ,				num_hidden_layers=self.num_hidden_layers       ,				num_attention_heads=self.num_attention_heads       ,				intermediate_size=self.intermediate_size       ,				hidden_act=self.hidden_act       ,				hidden_dropout_prob=self.hidden_dropout_prob       ,				attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,				max_position_embeddings=self.max_position_embeddings       ,				initializer_range=self.initializer_range       ,				)


				def   A_ (		self    :					Any       ,				lowercase_    :					Dict       ,				lowercase_    :					str       ,				lowercase_    :					Optional[Any]       ,				lowercase_    :					Union[str, Any]       ,				lowercase_    :					Dict       ,				lowercase_    :					Optional[int]					):
									snake_case_									=				MPNetModel(config=lowercase_					)
									model.to(lowercase_					)
									model.eval()
									snake_case_									=				model(lowercase_       ,				lowercase_					)
									snake_case_									=				model(lowercase_					)
									self.parent.assertEqual(result.last_hidden_state.shape       ,				(self.batch_size, self.seq_length, self.hidden_size)					)
									self.parent.assertEqual(result.pooler_output.shape       ,				(self.batch_size, self.hidden_size)					)


				def   A_ (		self    :					str       ,				lowercase_    :					Optional[Any]       ,				lowercase_    :					Any       ,				lowercase_    :					str       ,				lowercase_    :					Tuple       ,				lowercase_    :					Tuple       ,				lowercase_    :					Optional[int]					):
									snake_case_									=				MPNetForQuestionAnswering(config=lowercase_					)
									model.to(lowercase_					)
									model.eval()
									snake_case_									=				model(
									    lowercase_       ,				attention_mask=lowercase_       ,				start_positions=lowercase_       ,				end_positions=lowercase_       ,				)
									self.parent.assertEqual(result.start_logits.shape       ,				(self.batch_size, self.seq_length)					)
									self.parent.assertEqual(result.end_logits.shape       ,				(self.batch_size, self.seq_length)					)


				def   A_ (		self    :					Tuple       ,				lowercase_    :					Tuple       ,				lowercase_    :					List[str]       ,				lowercase_    :					str       ,				lowercase_    :					Optional[int]       ,				lowercase_    :					Optional[Any]       ,				lowercase_    :					Any					):
									snake_case_									=				self.num_labels
									snake_case_									=				MPNetForSequenceClassification(lowercase_					)
									model.to(lowercase_					)
									model.eval()
									snake_case_									=				model(lowercase_       ,				attention_mask=lowercase_       ,				labels=lowercase_					)
									self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.num_labels)					)


				def   A_ (		self    :					Any       ,				lowercase_    :					Any       ,				lowercase_    :					str       ,				lowercase_    :					Tuple       ,				lowercase_    :					List[str]       ,				lowercase_    :					Union[str, Any]       ,				lowercase_    :					Dict					):
									snake_case_									=				self.num_choices
									snake_case_									=				MPNetForMultipleChoice(config=lowercase_					)
									model.to(lowercase_					)
									model.eval()
									snake_case_									=				input_ids.unsqueeze(1					).expand(-1       ,				self.num_choices       ,				-1					).contiguous()
									snake_case_									=				input_mask.unsqueeze(1					).expand(-1       ,				self.num_choices       ,				-1					).contiguous()
									snake_case_									=				model(
									    lowercase_       ,				attention_mask=lowercase_       ,				labels=lowercase_       ,				)
									self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.num_choices)					)


				def   A_ (		self    :					Union[str, Any]       ,				lowercase_    :					List[Any]       ,				lowercase_    :					Optional[Any]       ,				lowercase_    :					Optional[int]       ,				lowercase_    :					str       ,				lowercase_    :					int       ,				lowercase_    :					List[str]					):
									snake_case_									=				self.num_labels
									snake_case_									=				MPNetForTokenClassification(config=lowercase_					)
									model.to(lowercase_					)
									model.eval()
									snake_case_									=				model(lowercase_       ,				attention_mask=lowercase_       ,				labels=lowercase_					)
									self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.seq_length, self.num_labels)					)


				def   A_ (		self    :					Union[str, Any]					):
									snake_case_									=				self.prepare_config_and_inputs()
									((snake_case_)			,(snake_case_)			,(snake_case_)			,(snake_case_)			,(snake_case_)			,(snake_case_))									=				config_and_inputs
									snake_case_									=				{'''input_ids''': input_ids, '''attention_mask''': input_mask}
									return config, inputs_dict







@require_torch
class 						a				(   _lowerCamelCase   ,     _lowerCamelCase   ,     unittest.TestCase     ):
				snake_case_      =		(
				    (
				        MPNetForMaskedLM,
				        MPNetForMultipleChoice,
				        MPNetForQuestionAnswering,
				        MPNetForSequenceClassification,
				        MPNetForTokenClassification,
				        MPNetModel,
				    )
				    if is_torch_available()
				    else ()
				)
				snake_case_      =		(
				    {
				        "feature-extraction": MPNetModel,
				        "fill-mask": MPNetForMaskedLM,
				        "question-answering": MPNetForQuestionAnswering,
				        "text-classification": MPNetForSequenceClassification,
				        "token-classification": MPNetForTokenClassification,
				        "zero-shot": MPNetForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				snake_case_      =		False
				snake_case_      =		True


				def   A_ (		self    :					Tuple					):
									snake_case_									=				MPNetModelTester(self					)
									snake_case_									=				ConfigTester(self       ,				config_class=lowercase_       ,				hidden_size=37					)


				def   A_ (		self    :					Union[str, Any]					):
									self.config_tester.run_common_tests()


				def   A_ (		self    :					Tuple					):
									snake_case_									=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mpnet_model(*lowercase_					)


				def   A_ (		self    :					List[Any]					):
									snake_case_									=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mpnet_for_sequence_classification(*lowercase_					)


				def   A_ (		self    :					List[Any]					):
									snake_case_									=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mpnet_for_multiple_choice(*lowercase_					)


				def   A_ (		self    :					Union[str, Any]					):
									snake_case_									=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mpnet_for_token_classification(*lowercase_					)


				def   A_ (		self    :					Tuple					):
									snake_case_									=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mpnet_for_question_answering(*lowercase_					)







@require_torch
class 						a				(   unittest.TestCase     ):


				@slow
				def   A_ (		self    :					List[Any]					):
									snake_case_									=				MPNetModel.from_pretrained('''microsoft/mpnet-base'''					)
									snake_case_									=				torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]					)
									snake_case_									=				model(lowercase_					)[0]
									snake_case_									=				torch.Size((1, 11, 768)					)
									self.assertEqual(output.shape       ,				lowercase_					)
									snake_case_									=				torch.tensor(
									    [[[-0.0550, 0.1943, -0.0740], [-0.0562, 0.2211, -0.0579], [-0.0437, 0.3337, -0.0641]]]					)
									# compare the actual values for a slice.
									self.assertTrue(torch.allclose(output[:, :3, :3]       ,				lowercase_       ,				atol=1e-4					)					)

