



from typing import TYPE_CHECKING

from ..utils import OptionalDependencyNotAvailable, _LazyModule, is_flax_available, is_tf_available, is_torch_available


UpperCAmelCase_			      =  {
    '''configuration_utils''': ['''GenerationConfig'''],
    '''streamers''': ['''TextIteratorStreamer''', '''TextStreamer'''],
}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCAmelCase_			      =  [
						    '''Constraint''',
						    '''ConstraintListState''',
						    '''DisjunctiveConstraint''',
						    '''PhrasalConstraint''',
						]
						UpperCAmelCase_			      =  [
						    '''BeamHypotheses''',
						    '''BeamScorer''',
						    '''BeamSearchScorer''',
						    '''ConstrainedBeamSearchScorer''',
						]
						UpperCAmelCase_			      =  [
						    '''EpsilonLogitsWarper''',
						    '''EtaLogitsWarper''',
						    '''ForcedBOSTokenLogitsProcessor''',
						    '''ForcedEOSTokenLogitsProcessor''',
						    '''HammingDiversityLogitsProcessor''',
						    '''InfNanRemoveLogitsProcessor''',
						    '''LogitsProcessor''',
						    '''LogitsProcessorList''',
						    '''LogitsWarper''',
						    '''MinLengthLogitsProcessor''',
						    '''MinNewTokensLengthLogitsProcessor''',
						    '''NoBadWordsLogitsProcessor''',
						    '''NoRepeatNGramLogitsProcessor''',
						    '''PrefixConstrainedLogitsProcessor''',
						    '''RepetitionPenaltyLogitsProcessor''',
						    '''SequenceBiasLogitsProcessor''',
						    '''EncoderRepetitionPenaltyLogitsProcessor''',
						    '''TemperatureLogitsWarper''',
						    '''TopKLogitsWarper''',
						    '''TopPLogitsWarper''',
						    '''TypicalLogitsWarper''',
						    '''EncoderNoRepeatNGramLogitsProcessor''',
						    '''ExponentialDecayLengthPenalty''',
						    '''LogitNormalization''',
						]
						UpperCAmelCase_			      =  [
						    '''MaxNewTokensCriteria''',
						    '''MaxLengthCriteria''',
						    '''MaxTimeCriteria''',
						    '''StoppingCriteria''',
						    '''StoppingCriteriaList''',
						    '''validate_stopping_criteria''',
						]
						UpperCAmelCase_			      =  [
						    '''GenerationMixin''',
						    '''top_k_top_p_filtering''',
						    '''GreedySearchEncoderDecoderOutput''',
						    '''GreedySearchDecoderOnlyOutput''',
						    '''SampleEncoderDecoderOutput''',
						    '''SampleDecoderOnlyOutput''',
						    '''BeamSearchEncoderDecoderOutput''',
						    '''BeamSearchDecoderOnlyOutput''',
						    '''BeamSampleEncoderDecoderOutput''',
						    '''BeamSampleDecoderOnlyOutput''',
						    '''ContrastiveSearchEncoderDecoderOutput''',
						    '''ContrastiveSearchDecoderOnlyOutput''',
						]

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCAmelCase_			      =  [
						    '''TFForcedBOSTokenLogitsProcessor''',
						    '''TFForcedEOSTokenLogitsProcessor''',
						    '''TFLogitsProcessor''',
						    '''TFLogitsProcessorList''',
						    '''TFLogitsWarper''',
						    '''TFMinLengthLogitsProcessor''',
						    '''TFNoBadWordsLogitsProcessor''',
						    '''TFNoRepeatNGramLogitsProcessor''',
						    '''TFRepetitionPenaltyLogitsProcessor''',
						    '''TFTemperatureLogitsWarper''',
						    '''TFTopKLogitsWarper''',
						    '''TFTopPLogitsWarper''',
						    '''TFForceTokensLogitsProcessor''',
						    '''TFSuppressTokensAtBeginLogitsProcessor''',
						    '''TFSuppressTokensLogitsProcessor''',
						]
						UpperCAmelCase_			      =  [
						    '''TFGenerationMixin''',
						    '''tf_top_k_top_p_filtering''',
						    '''TFGreedySearchDecoderOnlyOutput''',
						    '''TFGreedySearchEncoderDecoderOutput''',
						    '''TFSampleEncoderDecoderOutput''',
						    '''TFSampleDecoderOnlyOutput''',
						    '''TFBeamSearchEncoderDecoderOutput''',
						    '''TFBeamSearchDecoderOnlyOutput''',
						    '''TFBeamSampleEncoderDecoderOutput''',
						    '''TFBeamSampleDecoderOnlyOutput''',
						    '''TFContrastiveSearchEncoderDecoderOutput''',
						    '''TFContrastiveSearchDecoderOnlyOutput''',
						]

try:
						if not is_flax_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCAmelCase_			      =  [
						    '''FlaxForcedBOSTokenLogitsProcessor''',
						    '''FlaxForcedEOSTokenLogitsProcessor''',
						    '''FlaxLogitsProcessor''',
						    '''FlaxLogitsProcessorList''',
						    '''FlaxLogitsWarper''',
						    '''FlaxMinLengthLogitsProcessor''',
						    '''FlaxTemperatureLogitsWarper''',
						    '''FlaxTopKLogitsWarper''',
						    '''FlaxTopPLogitsWarper''',
						]
						UpperCAmelCase_			      =  [
						    '''FlaxGenerationMixin''',
						    '''FlaxGreedySearchOutput''',
						    '''FlaxSampleOutput''',
						    '''FlaxBeamSearchOutput''',
						]

if TYPE_CHECKING:
						from .configuration_utils import GenerationConfig
						from .streamers import TextIteratorStreamer, TextStreamer

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .beam_constraints import Constraint, ConstraintListState, DisjunctiveConstraint, PhrasalConstraint
												from .beam_search import BeamHypotheses, BeamScorer, BeamSearchScorer, ConstrainedBeamSearchScorer
												from .logits_process import (
												    EncoderNoRepeatNGramLogitsProcessor,
												    EncoderRepetitionPenaltyLogitsProcessor,
												    EpsilonLogitsWarper,
												    EtaLogitsWarper,
												    ExponentialDecayLengthPenalty,
												    ForcedBOSTokenLogitsProcessor,
												    ForcedEOSTokenLogitsProcessor,
												    HammingDiversityLogitsProcessor,
												    InfNanRemoveLogitsProcessor,
												    LogitNormalization,
												    LogitsProcessor,
												    LogitsProcessorList,
												    LogitsWarper,
												    MinLengthLogitsProcessor,
												    MinNewTokensLengthLogitsProcessor,
												    NoBadWordsLogitsProcessor,
												    NoRepeatNGramLogitsProcessor,
												    PrefixConstrainedLogitsProcessor,
												    RepetitionPenaltyLogitsProcessor,
												    SequenceBiasLogitsProcessor,
												    TemperatureLogitsWarper,
												    TopKLogitsWarper,
												    TopPLogitsWarper,
												    TypicalLogitsWarper,
												)
												from .stopping_criteria import (
												    MaxLengthCriteria,
												    MaxNewTokensCriteria,
												    MaxTimeCriteria,
												    StoppingCriteria,
												    StoppingCriteriaList,
												    validate_stopping_criteria,
												)
												from .utils import (
												    BeamSampleDecoderOnlyOutput,
												    BeamSampleEncoderDecoderOutput,
												    BeamSearchDecoderOnlyOutput,
												    BeamSearchEncoderDecoderOutput,
												    ContrastiveSearchDecoderOnlyOutput,
												    ContrastiveSearchEncoderDecoderOutput,
												    GenerationMixin,
												    GreedySearchDecoderOnlyOutput,
												    GreedySearchEncoderDecoderOutput,
												    SampleDecoderOnlyOutput,
												    SampleEncoderDecoderOutput,
												    top_k_top_p_filtering,
												)

						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tf_logits_process import (
												    TFForcedBOSTokenLogitsProcessor,
												    TFForcedEOSTokenLogitsProcessor,
												    TFForceTokensLogitsProcessor,
												    TFLogitsProcessor,
												    TFLogitsProcessorList,
												    TFLogitsWarper,
												    TFMinLengthLogitsProcessor,
												    TFNoBadWordsLogitsProcessor,
												    TFNoRepeatNGramLogitsProcessor,
												    TFRepetitionPenaltyLogitsProcessor,
												    TFSuppressTokensAtBeginLogitsProcessor,
												    TFSuppressTokensLogitsProcessor,
												    TFTemperatureLogitsWarper,
												    TFTopKLogitsWarper,
												    TFTopPLogitsWarper,
												)
												from .tf_utils import (
												    TFBeamSampleDecoderOnlyOutput,
												    TFBeamSampleEncoderDecoderOutput,
												    TFBeamSearchDecoderOnlyOutput,
												    TFBeamSearchEncoderDecoderOutput,
												    TFContrastiveSearchDecoderOnlyOutput,
												    TFContrastiveSearchEncoderDecoderOutput,
												    TFGenerationMixin,
												    TFGreedySearchDecoderOnlyOutput,
												    TFGreedySearchEncoderDecoderOutput,
												    TFSampleDecoderOnlyOutput,
												    TFSampleEncoderDecoderOutput,
												    tf_top_k_top_p_filtering,
												)

						try:
												if not is_flax_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .flax_logits_process import (
												    FlaxForcedBOSTokenLogitsProcessor,
												    FlaxForcedEOSTokenLogitsProcessor,
												    FlaxLogitsProcessor,
												    FlaxLogitsProcessorList,
												    FlaxLogitsWarper,
												    FlaxMinLengthLogitsProcessor,
												    FlaxTemperatureLogitsWarper,
												    FlaxTopKLogitsWarper,
												    FlaxTopPLogitsWarper,
												)
												from .flax_utils import FlaxBeamSearchOutput, FlaxGenerationMixin, FlaxGreedySearchOutput, FlaxSampleOutput
else:
						import sys

						UpperCAmelCase_			      =  _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)










import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
						from tokenizers import Tokenizer
						from tokenizers.models import WordPiece





class 		__SCREAMING_SNAKE_CASE      (				unittest.TestCase      ):






				"""simple docstring"""

				def 		__lowerCamelCase(				self						,	SCREAMING_SNAKE_CASE__      ):






					"""simple docstring"""

					_snake_case					:							Any					     =   list(tokenizer_class.max_model_input_sizes.keys()      )
					for model_name in sa_models[:1]:
						_snake_case					:							Optional[Any]					     =   tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__      )
						self.assertIsNotNone(SCREAMING_SNAKE_CASE__      )
						self.assertIsInstance(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )
						self.assertIsInstance(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )

						for special_tok in tokenizer.all_special_tokens:
							self.assertIsInstance(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )
							_snake_case					:							int					     =   tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__      )
							self.assertIsInstance(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )







				def 		__lowerCamelCase(				self						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__ = None      ):






					"""simple docstring"""

					_snake_case					:							int					     =   pickle.dumps(SCREAMING_SNAKE_CASE__      )
					self.assertIsNotNone(SCREAMING_SNAKE_CASE__      )

					_snake_case					:							Dict					     =   pickle.loads(SCREAMING_SNAKE_CASE__      )

					# Ensure is_fast is correctly restored
					self.assertEqual(be_restored.is_fast						,	be_original.is_fast      )

					# Ensure encodings are potentially correctly restored
					if be_original.is_fast:
						self.assertIsNotNone(be_restored.encodings      )
					else:
						self.assertIsNone(be_restored.encodings      )

					# Ensure the keys are the same
					for original_v, restored_v in zip(be_original.values()						,	be_restored.values()      ):
						if equal_op:
							self.assertTrue(equal_op(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )      )
						else:
							self.assertEqual(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )







				@slow
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					self.check_tokenizer_from_pretrained(SCREAMING_SNAKE_CASE__      )







				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					self.assertEqual(TensorType("""tf"""      )						,	TensorType.TENSORFLOW      )
					self.assertEqual(TensorType("""pt"""      )						,	TensorType.PYTORCH      )
					self.assertEqual(TensorType("""np"""      )						,	TensorType.NUMPY      )







				@require_tokenizers
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					import numpy as np

					_snake_case					:							Optional[int]					     =   BertTokenizer.from_pretrained("""bert-base-cased"""      )
					_snake_case					:							Any					     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""      )

					# Python no tensor
					with self.subTest("""BatchEncoding (Python, return_tensors=None)"""      ):
						self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""      )      )

					with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""      ):
						self.assert_dump_and_restore(
						    tokenizer_p("""Small example to encode"""						,	return_tensors=TensorType.NUMPY      )						,	np.array_equal      )

					with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""      ):
						self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""      )      )

					with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""      ):
						self.assert_dump_and_restore(
						    tokenizer_r("""Small example to encode"""						,	return_tensors=TensorType.NUMPY      )						,	np.array_equal      )







				@require_tf
				@require_tokenizers
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					import tensorflow as tf

					def tf_array_equals(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      ):
						return tf.reduce_all(tf.equal(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )      )

					_snake_case					:							int					     =   BertTokenizer.from_pretrained("""bert-base-cased"""      )
					_snake_case					:							str					     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""      )

					with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""      ):
						self.assert_dump_and_restore(
						    tokenizer_p("""Small example to encode"""						,	return_tensors=TensorType.TENSORFLOW      )						,	SCREAMING_SNAKE_CASE__      )

					with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""      ):
						self.assert_dump_and_restore(
						    tokenizer_r("""Small example to encode"""						,	return_tensors=TensorType.TENSORFLOW      )						,	SCREAMING_SNAKE_CASE__      )







				@require_torch
				@require_tokenizers
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					import torch

					_snake_case					:							Union[str, Any]					     =   BertTokenizer.from_pretrained("""bert-base-cased"""      )
					_snake_case					:							Dict					     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""      )

					with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""      ):
						self.assert_dump_and_restore(
						    tokenizer_p("""Small example to encode"""						,	return_tensors=TensorType.PYTORCH      )						,	torch.equal      )

					with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""      ):
						self.assert_dump_and_restore(
						    tokenizer_r("""Small example to encode"""						,	return_tensors=TensorType.PYTORCH      )						,	torch.equal      )







				@require_tokenizers
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							Optional[int]					     =   BertTokenizer.from_pretrained("""bert-base-cased"""      )
					_snake_case					:							int					     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""      )

					with self.subTest("""Python Tokenizer"""      ):
						self.assertFalse(tokenizer_p("""Small example to_encode"""      ).is_fast      )

					with self.subTest("""Rust Tokenizer"""      ):
						self.assertTrue(tokenizer_r("""Small example to_encode"""      ).is_fast      )







				@require_tokenizers
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							Optional[int]					     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""      )
					_snake_case					:							Dict					     =   tokenizer_r(["""Test""", """\xad""", """test"""]						,	is_split_into_words=SCREAMING_SNAKE_CASE__      )

					self.assertEqual(encoded.word_to_tokens(0      )						,	TokenSpan(start=1						,	end=2      )      )
					self.assertEqual(encoded.word_to_tokens(1      )						,	SCREAMING_SNAKE_CASE__      )
					self.assertEqual(encoded.word_to_tokens(2      )						,	TokenSpan(start=2						,	end=3      )      )







				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							List[Any]					     =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}      )
					_snake_case					:							Union[str, Any]					     =   batch.convert_to_tensors(tensor_type="""np"""      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(2, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(2,)      )
					# test converting the converted
					with CaptureStderr() as cs:
						_snake_case					:							Optional[int]					     =   batch.convert_to_tensors(tensor_type="""np"""      )
					self.assertFalse(len(cs.err      )						,	msg=f'''should have no warning, but got {cs.err}'''      )

					_snake_case					:							str					     =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}      )
					_snake_case					:							Optional[int]					     =   batch.convert_to_tensors(tensor_type="""np"""						,	prepend_batch_axis=SCREAMING_SNAKE_CASE__      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(1, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(1,)      )







				@require_torch
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							List[Any]					     =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}      )
					_snake_case					:							Union[str, Any]					     =   batch.convert_to_tensors(tensor_type="""pt"""      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(2, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(2,)      )
					# test converting the converted
					with CaptureStderr() as cs:
						_snake_case					:							Optional[Any]					     =   batch.convert_to_tensors(tensor_type="""pt"""      )
					self.assertFalse(len(cs.err      )						,	msg=f'''should have no warning, but got {cs.err}'''      )

					_snake_case					:							Union[str, Any]					     =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}      )
					_snake_case					:							Union[str, Any]					     =   batch.convert_to_tensors(tensor_type="""pt"""						,	prepend_batch_axis=SCREAMING_SNAKE_CASE__      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(1, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(1,)      )







				@require_tf
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							Optional[int]					     =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}      )
					_snake_case					:							str					     =   batch.convert_to_tensors(tensor_type="""tf"""      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(2, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(2,)      )
					# test converting the converted
					with CaptureStderr() as cs:
						_snake_case					:							int					     =   batch.convert_to_tensors(tensor_type="""tf"""      )
					self.assertFalse(len(cs.err      )						,	msg=f'''should have no warning, but got {cs.err}'''      )

					_snake_case					:							Optional[Any]					     =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}      )
					_snake_case					:							List[str]					     =   batch.convert_to_tensors(tensor_type="""tf"""						,	prepend_batch_axis=SCREAMING_SNAKE_CASE__      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(1, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(1,)      )







				@require_flax
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							List[Any]					     =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}      )
					_snake_case					:							Any					     =   batch.convert_to_tensors(tensor_type="""jax"""      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(2, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(2,)      )
					# test converting the converted
					with CaptureStderr() as cs:
						_snake_case					:							Dict					     =   batch.convert_to_tensors(tensor_type="""jax"""      )
					self.assertFalse(len(cs.err      )						,	msg=f'''should have no warning, but got {cs.err}'''      )

					_snake_case					:							List[str]					     =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}      )
					_snake_case					:							List[Any]					     =   batch.convert_to_tensors(tensor_type="""jax"""						,	prepend_batch_axis=SCREAMING_SNAKE_CASE__      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(1, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(1,)      )







				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							Any					     =   [{"""input_ids""": np.array([0, 1, 2]      )}, {"""input_ids""": np.array([0, 1, 2, 3]      )}]
					_snake_case					:							int					     =   BertTokenizer.from_pretrained("""bert-base-cased"""      )

					_snake_case					:							List[Any]					     =   tokenizer.pad(SCREAMING_SNAKE_CASE__						,	padding=SCREAMING_SNAKE_CASE__      )
					self.assertTrue(isinstance(batch["""input_ids"""]						,	np.ndarray      )      )
					self.assertEqual(batch["""input_ids"""].tolist()						,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )
					_snake_case					:							Optional[int]					     =   tokenizer.pad(SCREAMING_SNAKE_CASE__						,	padding=SCREAMING_SNAKE_CASE__						,	return_tensors="""np"""      )
					self.assertTrue(isinstance(batch["""input_ids"""]						,	np.ndarray      )      )
					self.assertEqual(batch["""input_ids"""].tolist()						,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )







				@require_torch
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					import torch

					_snake_case					:							str					     =   [{"""input_ids""": torch.tensor([0, 1, 2]      )}, {"""input_ids""": torch.tensor([0, 1, 2, 3]      )}]
					_snake_case					:							Tuple					     =   BertTokenizer.from_pretrained("""bert-base-cased"""      )

					_snake_case					:							Dict					     =   tokenizer.pad(SCREAMING_SNAKE_CASE__						,	padding=SCREAMING_SNAKE_CASE__      )
					self.assertTrue(isinstance(batch["""input_ids"""]						,	torch.Tensor      )      )
					self.assertEqual(batch["""input_ids"""].tolist()						,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )
					_snake_case					:							Tuple					     =   tokenizer.pad(SCREAMING_SNAKE_CASE__						,	padding=SCREAMING_SNAKE_CASE__						,	return_tensors="""pt"""      )
					self.assertTrue(isinstance(batch["""input_ids"""]						,	torch.Tensor      )      )
					self.assertEqual(batch["""input_ids"""].tolist()						,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )







				@require_tf
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					import tensorflow as tf

					_snake_case					:							Optional[Any]					     =   [{"""input_ids""": tf.constant([0, 1, 2]      )}, {"""input_ids""": tf.constant([0, 1, 2, 3]      )}]
					_snake_case					:							Optional[Any]					     =   BertTokenizer.from_pretrained("""bert-base-cased"""      )

					_snake_case					:							Optional[Any]					     =   tokenizer.pad(SCREAMING_SNAKE_CASE__						,	padding=SCREAMING_SNAKE_CASE__      )
					self.assertTrue(isinstance(batch["""input_ids"""]						,	tf.Tensor      )      )
					self.assertEqual(batch["""input_ids"""].numpy().tolist()						,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )
					_snake_case					:							List[Any]					     =   tokenizer.pad(SCREAMING_SNAKE_CASE__						,	padding=SCREAMING_SNAKE_CASE__						,	return_tensors="""tf"""      )
					self.assertTrue(isinstance(batch["""input_ids"""]						,	tf.Tensor      )      )
					self.assertEqual(batch["""input_ids"""].numpy().tolist()						,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )







				@require_tokenizers
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							Optional[int]					     =   Tokenizer(WordPiece(unk_token="""[UNK]"""      )      )
					PreTrainedTokenizerFast(tokenizer_object=SCREAMING_SNAKE_CASE__      )







				@require_tokenizers
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							Optional[Any]					     =   Tokenizer(WordPiece(unk_token="""[UNK]"""      )      )
					with tempfile.TemporaryDirectory() as tmpdirname:
						bert_tokenizer.save(os.path.join(SCREAMING_SNAKE_CASE__						,	"""tokenizer.json"""      )      )
						PreTrainedTokenizerFast(tokenizer_file=os.path.join(SCREAMING_SNAKE_CASE__						,	"""tokenizer.json"""      )      )





