



from __future__ import annotations

import unittest

from transformers import MPNetConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers.models.mpnet.modeling_tf_mpnet import (
					    TFMPNetForMaskedLM,
					    TFMPNetForMultipleChoice,
					    TFMPNetForQuestionAnswering,
					    TFMPNetForSequenceClassification,
					    TFMPNetForTokenClassification,
					    TFMPNetModel,
					)





class 							lowerCAmelCase_		:







		def __init__(    self  :   List[str]       ,		UpperCAmelCase_  :   int       ,		UpperCAmelCase_  :   Tuple=13       ,		UpperCAmelCase_  :   Optional[int]=7       ,		UpperCAmelCase_  :   int=True       ,		UpperCAmelCase_  :   List[Any]=True       ,		UpperCAmelCase_  :   Optional[int]=False       ,		UpperCAmelCase_  :   Any=True       ,		UpperCAmelCase_  :   int=99       ,		UpperCAmelCase_  :   Union[str, Any]=64       ,		UpperCAmelCase_  :   Optional[Any]=2       ,		UpperCAmelCase_  :   List[str]=4       ,		UpperCAmelCase_  :   Optional[Any]=64       ,		UpperCAmelCase_  :   Union[str, Any]="gelu"       ,		UpperCAmelCase_  :   Tuple=0.1       ,		UpperCAmelCase_  :   List[Any]=0.1       ,		UpperCAmelCase_  :   Any=512       ,		UpperCAmelCase_  :   Union[str, Any]=16       ,		UpperCAmelCase_  :   Union[str, Any]=2       ,		UpperCAmelCase_  :   List[str]=0.02       ,		UpperCAmelCase_  :   Any=3       ,		UpperCAmelCase_  :   Optional[Any]=4       ,		UpperCAmelCase_  :   str=None       ,		)       ->		Dict:


						'''simple docstring'''







						_UpperCAmelCase						:  str     					=	parent
						_UpperCAmelCase						:  Any     					=	batch_size
						_UpperCAmelCase						:  str     					=	seq_length
						_UpperCAmelCase						:  List[str]     					=	is_training
						_UpperCAmelCase						:  Tuple     					=	use_input_mask
						_UpperCAmelCase						:  List[Any]     					=	use_token_type_ids
						_UpperCAmelCase						:  List[Any]     					=	use_labels
						_UpperCAmelCase						:  Optional[int]     					=	vocab_size
						_UpperCAmelCase						:  Any     					=	hidden_size
						_UpperCAmelCase						:  Optional[int]     					=	num_hidden_layers
						_UpperCAmelCase						:  Optional[int]     					=	num_attention_heads
						_UpperCAmelCase						:  Optional[Any]     					=	intermediate_size
						_UpperCAmelCase						:  Any     					=	hidden_act
						_UpperCAmelCase						:  Dict     					=	hidden_dropout_prob
						_UpperCAmelCase						:  Union[str, Any]     					=	attention_probs_dropout_prob
						_UpperCAmelCase						:  Union[str, Any]     					=	max_position_embeddings
						_UpperCAmelCase						:  str     					=	type_vocab_size
						_UpperCAmelCase						:  List[str]     					=	type_sequence_label_size
						_UpperCAmelCase						:  List[str]     					=	initializer_range
						_UpperCAmelCase						:  List[str]     					=	num_labels
						_UpperCAmelCase						:  List[Any]     					=	num_choices
						_UpperCAmelCase						:  List[Any]     					=	scope







		def 					a_  (    self  :   Optional[Any]					)       ->		Optional[Any]:


						'''simple docstring'''







						_UpperCAmelCase						:  Optional[int]     					=	ids_tensor([self.batch_size, self.seq_length]       ,		self.vocab_size					)

						_UpperCAmelCase						:  Optional[int]     					=	None
						if self.use_input_mask:
										_UpperCAmelCase						:  Union[str, Any]     					=	random_attention_mask([self.batch_size, self.seq_length]					)

						_UpperCAmelCase						:  Optional[Any]     					=	None
						_UpperCAmelCase						:  int     					=	None
						_UpperCAmelCase						:  List[str]     					=	None
						if self.use_labels:
										_UpperCAmelCase						:  List[str]     					=	ids_tensor([self.batch_size]       ,		self.type_sequence_label_size					)
										_UpperCAmelCase						:  Tuple     					=	ids_tensor([self.batch_size, self.seq_length]       ,		self.num_labels					)
										_UpperCAmelCase						:  List[Any]     					=	ids_tensor([self.batch_size]       ,		self.num_choices					)

						_UpperCAmelCase						:  Dict     					=	MPNetConfig(
						    vocab_size=self.vocab_size       ,		hidden_size=self.hidden_size       ,		num_hidden_layers=self.num_hidden_layers       ,		num_attention_heads=self.num_attention_heads       ,		intermediate_size=self.intermediate_size       ,		hidden_act=self.hidden_act       ,		hidden_dropout_prob=self.hidden_dropout_prob       ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,		max_position_embeddings=self.max_position_embeddings       ,		initializer_range=self.initializer_range       ,		)
						return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels







		def 					a_  (    self  :   Any       ,		UpperCAmelCase_  :   Dict       ,		UpperCAmelCase_  :   Tuple       ,		UpperCAmelCase_  :   Optional[int]       ,		UpperCAmelCase_  :   Optional[int]       ,		UpperCAmelCase_  :   int       ,		UpperCAmelCase_  :   List[str]					)       ->		List[Any]:


						'''simple docstring'''







						_UpperCAmelCase						:  List[str]     					=	TFMPNetModel(config=UpperCAmelCase_					)
						_UpperCAmelCase						:  int     					=	{"""input_ids""": input_ids, """attention_mask""": input_mask}
						_UpperCAmelCase						:  Any     					=	model(UpperCAmelCase_					)
						_UpperCAmelCase						:  Optional[int]     					=	[input_ids, input_mask]
						_UpperCAmelCase						:  Tuple     					=	model(UpperCAmelCase_					)
						self.parent.assertEqual(result.last_hidden_state.shape       ,		(self.batch_size, self.seq_length, self.hidden_size)					)







		def 					a_  (    self  :   Dict       ,		UpperCAmelCase_  :   Tuple       ,		UpperCAmelCase_  :   List[str]       ,		UpperCAmelCase_  :   List[str]       ,		UpperCAmelCase_  :   Tuple       ,		UpperCAmelCase_  :   Dict       ,		UpperCAmelCase_  :   str					)       ->		Optional[Any]:


						'''simple docstring'''







						_UpperCAmelCase						:  Dict     					=	TFMPNetForMaskedLM(config=UpperCAmelCase_					)
						_UpperCAmelCase						:  Union[str, Any]     					=	{"""input_ids""": input_ids, """attention_mask""": input_mask}
						_UpperCAmelCase						:  str     					=	model(UpperCAmelCase_					)
						self.parent.assertEqual(result.logits.shape       ,		(self.batch_size, self.seq_length, self.vocab_size)					)







		def 					a_  (    self  :   str       ,		UpperCAmelCase_  :   Optional[int]       ,		UpperCAmelCase_  :   int       ,		UpperCAmelCase_  :   Union[str, Any]       ,		UpperCAmelCase_  :   Any       ,		UpperCAmelCase_  :   List[Any]       ,		UpperCAmelCase_  :   Optional[Any]					)       ->		Optional[int]:


						'''simple docstring'''







						_UpperCAmelCase						:  Optional[Any]     					=	TFMPNetForQuestionAnswering(config=UpperCAmelCase_					)
						_UpperCAmelCase						:  Dict     					=	{
						    """input_ids""": input_ids,
						    """attention_mask""": input_mask,
						}
						_UpperCAmelCase						:  List[str]     					=	model(UpperCAmelCase_					)
						self.parent.assertEqual(result.start_logits.shape       ,		(self.batch_size, self.seq_length)					)
						self.parent.assertEqual(result.end_logits.shape       ,		(self.batch_size, self.seq_length)					)







		def 					a_  (    self  :   Dict       ,		UpperCAmelCase_  :   List[str]       ,		UpperCAmelCase_  :   List[Any]       ,		UpperCAmelCase_  :   List[str]       ,		UpperCAmelCase_  :   Any       ,		UpperCAmelCase_  :   Union[str, Any]       ,		UpperCAmelCase_  :   Optional[int]					)       ->		Union[str, Any]:


						'''simple docstring'''







						_UpperCAmelCase						:  str     					=	self.num_labels
						_UpperCAmelCase						:  int     					=	TFMPNetForSequenceClassification(UpperCAmelCase_					)
						_UpperCAmelCase						:  Union[str, Any]     					=	{"""input_ids""": input_ids, """attention_mask""": input_mask}
						_UpperCAmelCase						:  Union[str, Any]     					=	model(UpperCAmelCase_					)
						self.parent.assertEqual(result.logits.shape       ,		(self.batch_size, self.num_labels)					)







		def 					a_  (    self  :   Dict       ,		UpperCAmelCase_  :   str       ,		UpperCAmelCase_  :   Union[str, Any]       ,		UpperCAmelCase_  :   Optional[int]       ,		UpperCAmelCase_  :   Union[str, Any]       ,		UpperCAmelCase_  :   int       ,		UpperCAmelCase_  :   str					)       ->		List[str]:


						'''simple docstring'''







						_UpperCAmelCase						:  str     					=	self.num_choices
						_UpperCAmelCase						:  Optional[Any]     					=	TFMPNetForMultipleChoice(UpperCAmelCase_					)
						_UpperCAmelCase						:  Tuple     					=	tf.tile(tf.expand_dims(UpperCAmelCase_       ,		1					)       ,		(1, self.num_choices, 1)					)
						_UpperCAmelCase						:  List[str]     					=	tf.tile(tf.expand_dims(UpperCAmelCase_       ,		1					)       ,		(1, self.num_choices, 1)					)
						_UpperCAmelCase						:  Any     					=	{
						    """input_ids""": multiple_choice_inputs_ids,
						    """attention_mask""": multiple_choice_input_mask,
						}
						_UpperCAmelCase						:  Union[str, Any]     					=	model(UpperCAmelCase_					)
						self.parent.assertEqual(result.logits.shape       ,		(self.batch_size, self.num_choices)					)







		def 					a_  (    self  :   str       ,		UpperCAmelCase_  :   Optional[Any]       ,		UpperCAmelCase_  :   Tuple       ,		UpperCAmelCase_  :   Optional[int]       ,		UpperCAmelCase_  :   Tuple       ,		UpperCAmelCase_  :   List[str]       ,		UpperCAmelCase_  :   str					)       ->		List[Any]:


						'''simple docstring'''







						_UpperCAmelCase						:  Dict     					=	self.num_labels
						_UpperCAmelCase						:  Tuple     					=	TFMPNetForTokenClassification(UpperCAmelCase_					)
						_UpperCAmelCase						:  List[Any]     					=	{"""input_ids""": input_ids, """attention_mask""": input_mask}
						_UpperCAmelCase						:  List[str]     					=	model(UpperCAmelCase_					)
						self.parent.assertEqual(result.logits.shape       ,		(self.batch_size, self.seq_length, self.num_labels)					)







		def 					a_  (    self  :   List[str]					)       ->		Dict:


						'''simple docstring'''







						_UpperCAmelCase						:  int     					=	self.prepare_config_and_inputs()
						(_UpperCAmelCase)						:  Optional[int]     					=	config_and_inputs
						_UpperCAmelCase						:  Optional[int]     					=	{"""input_ids""": input_ids, """attention_mask""": input_mask}
						return config, inputs_dict





@require_tf
class 							lowerCAmelCase_		(		lowercase_							,		lowercase_							,		unittest.TestCase     ):
		SCREAMING_SNAKE_CASE_       :			str					       =	(
		    (
		        TFMPNetForMaskedLM,
		        TFMPNetForMultipleChoice,
		        TFMPNetForQuestionAnswering,
		        TFMPNetForSequenceClassification,
		        TFMPNetForTokenClassification,
		        TFMPNetModel,
		    )
		    if is_tf_available()
		    else ()
		)
		SCREAMING_SNAKE_CASE_       :			str					       =	(
		    {
		        "feature-extraction": TFMPNetModel,
		        "fill-mask": TFMPNetForMaskedLM,
		        "question-answering": TFMPNetForQuestionAnswering,
		        "text-classification": TFMPNetForSequenceClassification,
		        "token-classification": TFMPNetForTokenClassification,
		        "zero-shot": TFMPNetForSequenceClassification,
		    }
		    if is_tf_available()
		    else {}
		)
		SCREAMING_SNAKE_CASE_       :			List[str]					       =	False
		SCREAMING_SNAKE_CASE_       :			str					       =	False







		def 					a_  (    self  :   List[str]					)       ->		List[str]:


						'''simple docstring'''







						_UpperCAmelCase						:  List[Any]     					=	TFMPNetModelTester(self					)
						_UpperCAmelCase						:  Tuple     					=	ConfigTester(self       ,		config_class=UpperCAmelCase_       ,		hidden_size=37					)







		def 					a_  (    self  :   Dict					)       ->		Any:


						'''simple docstring'''







						self.config_tester.run_common_tests()







		def 					a_  (    self  :   Optional[int]					)       ->		Optional[int]:


						'''simple docstring'''







						_UpperCAmelCase						:  Dict     					=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_mpnet_model(*UpperCAmelCase_					)







		def 					a_  (    self  :   int					)       ->		Any:


						'''simple docstring'''







						_UpperCAmelCase						:  Dict     					=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_mpnet_for_masked_lm(*UpperCAmelCase_					)







		def 					a_  (    self  :   Optional[Any]					)       ->		Optional[int]:


						'''simple docstring'''







						_UpperCAmelCase						:  str     					=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_mpnet_for_question_answering(*UpperCAmelCase_					)







		def 					a_  (    self  :   int					)       ->		Tuple:


						'''simple docstring'''







						_UpperCAmelCase						:  List[Any]     					=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_mpnet_for_sequence_classification(*UpperCAmelCase_					)







		def 					a_  (    self  :   Optional[Any]					)       ->		Dict:


						'''simple docstring'''







						_UpperCAmelCase						:  List[str]     					=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_mpnet_for_multiple_choice(*UpperCAmelCase_					)







		def 					a_  (    self  :   List[Any]					)       ->		Optional[Any]:


						'''simple docstring'''







						_UpperCAmelCase						:  Dict     					=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_mpnet_for_token_classification(*UpperCAmelCase_					)







		@slow
		def 					a_  (    self  :   str					)       ->		Optional[Any]:


						'''simple docstring'''







						for model_name in ["microsoft/mpnet-base"]:
										_UpperCAmelCase						:  Optional[Any]     					=	TFMPNetModel.from_pretrained(UpperCAmelCase_					)
										self.assertIsNotNone(UpperCAmelCase_					)





@require_tf
class 							lowerCAmelCase_		(		unittest.TestCase     ):







		@slow
		def 					a_  (    self  :   Any					)       ->		Dict:


						'''simple docstring'''







						_UpperCAmelCase						:  str     					=	TFMPNetModel.from_pretrained('''microsoft/mpnet-base'''					)
						_UpperCAmelCase						:  str     					=	tf.constant([[0, 1, 2, 3, 4, 5]]					)
						_UpperCAmelCase						:  Tuple     					=	model(UpperCAmelCase_					)[0]

						_UpperCAmelCase						:  Optional[int]     					=	[1, 6, 768]
						self.assertEqual(output.shape       ,		UpperCAmelCase_					)

						_UpperCAmelCase						:  Dict     					=	tf.constant(
						    [
						        [
						            [-0.1_067_172, 0.08_216_473, 0.0_024_543],
						            [-0.03_465_879, 0.8_354_118, -0.03_252_288],
						            [-0.06_569_476, -0.12_424_111, -0.0_494_436],
						        ]
						    ]					)
						tf.debugging.assert_near(output[:, :3, :3]       ,		UpperCAmelCase_       ,		atol=1E-4					)





"""simple docstring"""







import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
		from tokenizers import Tokenizer
		from tokenizers.models import WordPiece





class   UpperCamelCase	(     unittest.TestCase       ):




	"""simple docstring"""






	def    lowerCamelCase__					(     self				,UpperCAmelCase_   ):
			_lowercase				: Optional[int]     =   list(tokenizer_class.max_model_input_sizes.keys()   )
			for model_name in sa_models[:1]:
					_lowercase				: Any     =   tokenizer_class.from_pretrained(UpperCAmelCase_   )
					self.assertIsNotNone(UpperCAmelCase_   )
					self.assertIsInstance(UpperCAmelCase_				,UpperCAmelCase_   )
					self.assertIsInstance(UpperCAmelCase_				,UpperCAmelCase_   )

					for special_tok in tokenizer.all_special_tokens:
							self.assertIsInstance(UpperCAmelCase_				,UpperCAmelCase_   )
							_lowercase				: Any     =   tokenizer.convert_tokens_to_ids(UpperCAmelCase_   )
							self.assertIsInstance(UpperCAmelCase_				,UpperCAmelCase_   )






	def    lowerCamelCase__					(     self				,UpperCAmelCase_				,UpperCAmelCase_ = None   ):
			_lowercase				: Any     =   pickle.dumps(UpperCAmelCase_   )
			self.assertIsNotNone(UpperCAmelCase_   )

			_lowercase				: Optional[int]     =   pickle.loads(UpperCAmelCase_   )

			# Ensure is_fast is correctly restored
			self.assertEqual(be_restored.is_fast				,be_original.is_fast   )

			# Ensure encodings are potentially correctly restored
			if be_original.is_fast:
					self.assertIsNotNone(be_restored.encodings   )
			else:
					self.assertIsNone(be_restored.encodings   )

			# Ensure the keys are the same
			for original_v, restored_v in zip(be_original.values()				,be_restored.values()   ):
					if equal_op:
							self.assertTrue(equal_op(UpperCAmelCase_				,UpperCAmelCase_   )   )
					else:
							self.assertEqual(UpperCAmelCase_				,UpperCAmelCase_   )






	@slow
	def    lowerCamelCase__					(     self   ):
			self.check_tokenizer_from_pretrained(UpperCAmelCase_   )






	def    lowerCamelCase__					(     self   ):
			self.assertEqual(TensorType("""tf"""   )				,TensorType.TENSORFLOW   )
			self.assertEqual(TensorType("""pt"""   )				,TensorType.PYTORCH   )
			self.assertEqual(TensorType("""np"""   )				,TensorType.NUMPY   )






	@require_tokenizers
	def    lowerCamelCase__					(     self   ):
			import numpy as np

			_lowercase				: Optional[Any]     =   BertTokenizer.from_pretrained("""bert-base-cased"""   )
			_lowercase				: Dict     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""   )

			# Python no tensor
			with self.subTest("""BatchEncoding (Python, return_tensors=None)"""   ):
					self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""   )   )

			with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""   ):
					self.assert_dump_and_restore(
					    tokenizer_p("""Small example to encode"""				,return_tensors=TensorType.NUMPY   )				,np.array_equal   )

			with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""   ):
					self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""   )   )

			with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""   ):
					self.assert_dump_and_restore(
					    tokenizer_r("""Small example to encode"""				,return_tensors=TensorType.NUMPY   )				,np.array_equal   )






	@require_tf
	@require_tokenizers
	def    lowerCamelCase__					(     self   ):
			import tensorflow as tf

			def tf_array_equals(UpperCAmelCase_				,UpperCAmelCase_   ):
					return tf.reduce_all(tf.equal(UpperCAmelCase_				,UpperCAmelCase_   )   )

			_lowercase				: Optional[int]     =   BertTokenizer.from_pretrained("""bert-base-cased"""   )
			_lowercase				: Any     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""   )

			with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""   ):
					self.assert_dump_and_restore(
					    tokenizer_p("""Small example to encode"""				,return_tensors=TensorType.TENSORFLOW   )				,UpperCAmelCase_   )

			with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""   ):
					self.assert_dump_and_restore(
					    tokenizer_r("""Small example to encode"""				,return_tensors=TensorType.TENSORFLOW   )				,UpperCAmelCase_   )






	@require_torch
	@require_tokenizers
	def    lowerCamelCase__					(     self   ):
			import torch

			_lowercase				: Union[str, Any]     =   BertTokenizer.from_pretrained("""bert-base-cased"""   )
			_lowercase				: int     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""   )

			with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""   ):
					self.assert_dump_and_restore(
					    tokenizer_p("""Small example to encode"""				,return_tensors=TensorType.PYTORCH   )				,torch.equal   )

			with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""   ):
					self.assert_dump_and_restore(
					    tokenizer_r("""Small example to encode"""				,return_tensors=TensorType.PYTORCH   )				,torch.equal   )






	@require_tokenizers
	def    lowerCamelCase__					(     self   ):
			_lowercase				: int     =   BertTokenizer.from_pretrained("""bert-base-cased"""   )
			_lowercase				: Tuple     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""   )

			with self.subTest("""Python Tokenizer"""   ):
					self.assertFalse(tokenizer_p("""Small example to_encode"""   ).is_fast   )

			with self.subTest("""Rust Tokenizer"""   ):
					self.assertTrue(tokenizer_r("""Small example to_encode"""   ).is_fast   )






	@require_tokenizers
	def    lowerCamelCase__					(     self   ):
			_lowercase				: Optional[Any]     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""   )
			_lowercase				: Optional[int]     =   tokenizer_r(["""Test""", """\xad""", """test"""]				,is_split_into_words=UpperCAmelCase_   )

			self.assertEqual(encoded.word_to_tokens(0   )				,TokenSpan(start=1				,end=2   )   )
			self.assertEqual(encoded.word_to_tokens(1   )				,UpperCAmelCase_   )
			self.assertEqual(encoded.word_to_tokens(2   )				,TokenSpan(start=2				,end=3   )   )






	def    lowerCamelCase__					(     self   ):
			_lowercase				: Tuple     =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}   )
			_lowercase				: Any     =   batch.convert_to_tensors(tensor_type="""np"""   )
			self.assertEqual(tensor_batch["""inputs"""].shape				,(2, 3)   )
			self.assertEqual(tensor_batch["""labels"""].shape				,(2,)   )
			# test converting the converted
			with CaptureStderr() as cs:
					_lowercase				: Tuple     =   batch.convert_to_tensors(tensor_type="""np"""   )
			self.assertFalse(len(cs.err   )				,msg=f"""should have no warning, but got {cs.err}"""   )

			_lowercase				: List[str]     =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}   )
			_lowercase				: Any     =   batch.convert_to_tensors(tensor_type="""np"""				,prepend_batch_axis=UpperCAmelCase_   )
			self.assertEqual(tensor_batch["""inputs"""].shape				,(1, 3)   )
			self.assertEqual(tensor_batch["""labels"""].shape				,(1,)   )






	@require_torch
	def    lowerCamelCase__					(     self   ):
			_lowercase				: str     =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}   )
			_lowercase				: Any     =   batch.convert_to_tensors(tensor_type="""pt"""   )
			self.assertEqual(tensor_batch["""inputs"""].shape				,(2, 3)   )
			self.assertEqual(tensor_batch["""labels"""].shape				,(2,)   )
			# test converting the converted
			with CaptureStderr() as cs:
					_lowercase				: str     =   batch.convert_to_tensors(tensor_type="""pt"""   )
			self.assertFalse(len(cs.err   )				,msg=f"""should have no warning, but got {cs.err}"""   )

			_lowercase				: Dict     =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}   )
			_lowercase				: List[str]     =   batch.convert_to_tensors(tensor_type="""pt"""				,prepend_batch_axis=UpperCAmelCase_   )
			self.assertEqual(tensor_batch["""inputs"""].shape				,(1, 3)   )
			self.assertEqual(tensor_batch["""labels"""].shape				,(1,)   )






	@require_tf
	def    lowerCamelCase__					(     self   ):
			_lowercase				: int     =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}   )
			_lowercase				: List[str]     =   batch.convert_to_tensors(tensor_type="""tf"""   )
			self.assertEqual(tensor_batch["""inputs"""].shape				,(2, 3)   )
			self.assertEqual(tensor_batch["""labels"""].shape				,(2,)   )
			# test converting the converted
			with CaptureStderr() as cs:
					_lowercase				: Tuple     =   batch.convert_to_tensors(tensor_type="""tf"""   )
			self.assertFalse(len(cs.err   )				,msg=f"""should have no warning, but got {cs.err}"""   )

			_lowercase				: Tuple     =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}   )
			_lowercase				: List[str]     =   batch.convert_to_tensors(tensor_type="""tf"""				,prepend_batch_axis=UpperCAmelCase_   )
			self.assertEqual(tensor_batch["""inputs"""].shape				,(1, 3)   )
			self.assertEqual(tensor_batch["""labels"""].shape				,(1,)   )






	@require_flax
	def    lowerCamelCase__					(     self   ):
			_lowercase				: Any     =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}   )
			_lowercase				: Tuple     =   batch.convert_to_tensors(tensor_type="""jax"""   )
			self.assertEqual(tensor_batch["""inputs"""].shape				,(2, 3)   )
			self.assertEqual(tensor_batch["""labels"""].shape				,(2,)   )
			# test converting the converted
			with CaptureStderr() as cs:
					_lowercase				: str     =   batch.convert_to_tensors(tensor_type="""jax"""   )
			self.assertFalse(len(cs.err   )				,msg=f"""should have no warning, but got {cs.err}"""   )

			_lowercase				: Union[str, Any]     =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}   )
			_lowercase				: List[Any]     =   batch.convert_to_tensors(tensor_type="""jax"""				,prepend_batch_axis=UpperCAmelCase_   )
			self.assertEqual(tensor_batch["""inputs"""].shape				,(1, 3)   )
			self.assertEqual(tensor_batch["""labels"""].shape				,(1,)   )






	def    lowerCamelCase__					(     self   ):
			_lowercase				: Optional[int]     =   [{"""input_ids""": np.array([0, 1, 2]   )}, {"""input_ids""": np.array([0, 1, 2, 3]   )}]
			_lowercase				: Optional[int]     =   BertTokenizer.from_pretrained("""bert-base-cased"""   )

			_lowercase				: List[str]     =   tokenizer.pad(UpperCAmelCase_				,padding=UpperCAmelCase_   )
			self.assertTrue(isinstance(batch["""input_ids"""]				,np.ndarray   )   )
			self.assertEqual(batch["""input_ids"""].tolist()				,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )
			_lowercase				: Optional[int]     =   tokenizer.pad(UpperCAmelCase_				,padding=UpperCAmelCase_				,return_tensors="""np"""   )
			self.assertTrue(isinstance(batch["""input_ids"""]				,np.ndarray   )   )
			self.assertEqual(batch["""input_ids"""].tolist()				,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )






	@require_torch
	def    lowerCamelCase__					(     self   ):
			import torch

			_lowercase				: List[Any]     =   [{"""input_ids""": torch.tensor([0, 1, 2]   )}, {"""input_ids""": torch.tensor([0, 1, 2, 3]   )}]
			_lowercase				: Tuple     =   BertTokenizer.from_pretrained("""bert-base-cased"""   )

			_lowercase				: List[Any]     =   tokenizer.pad(UpperCAmelCase_				,padding=UpperCAmelCase_   )
			self.assertTrue(isinstance(batch["""input_ids"""]				,torch.Tensor   )   )
			self.assertEqual(batch["""input_ids"""].tolist()				,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )
			_lowercase				: List[Any]     =   tokenizer.pad(UpperCAmelCase_				,padding=UpperCAmelCase_				,return_tensors="""pt"""   )
			self.assertTrue(isinstance(batch["""input_ids"""]				,torch.Tensor   )   )
			self.assertEqual(batch["""input_ids"""].tolist()				,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )






	@require_tf
	def    lowerCamelCase__					(     self   ):
			import tensorflow as tf

			_lowercase				: List[Any]     =   [{"""input_ids""": tf.constant([0, 1, 2]   )}, {"""input_ids""": tf.constant([0, 1, 2, 3]   )}]
			_lowercase				: Dict     =   BertTokenizer.from_pretrained("""bert-base-cased"""   )

			_lowercase				: Optional[int]     =   tokenizer.pad(UpperCAmelCase_				,padding=UpperCAmelCase_   )
			self.assertTrue(isinstance(batch["""input_ids"""]				,tf.Tensor   )   )
			self.assertEqual(batch["""input_ids"""].numpy().tolist()				,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )
			_lowercase				: str     =   tokenizer.pad(UpperCAmelCase_				,padding=UpperCAmelCase_				,return_tensors="""tf"""   )
			self.assertTrue(isinstance(batch["""input_ids"""]				,tf.Tensor   )   )
			self.assertEqual(batch["""input_ids"""].numpy().tolist()				,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )






	@require_tokenizers
	def    lowerCamelCase__					(     self   ):
			_lowercase				: Tuple     =   Tokenizer(WordPiece(unk_token="""[UNK]"""   )   )
			PreTrainedTokenizerFast(tokenizer_object=UpperCAmelCase_   )






	@require_tokenizers
	def    lowerCamelCase__					(     self   ):
			_lowercase				: List[Any]     =   Tokenizer(WordPiece(unk_token="""[UNK]"""   )   )
			with tempfile.TemporaryDirectory() as tmpdirname:
					bert_tokenizer.save(os.path.join(UpperCAmelCase_				,"""tokenizer.json"""   )   )
					PreTrainedTokenizerFast(tokenizer_file=os.path.join(UpperCAmelCase_				,"""tokenizer.json"""   )   )



