






'''simple docstring'''

from __future__ import annotations

import inspect
import unittest

import numpy as np

from transformers import SwinConfig
from transformers.testing_utils import require_tf, require_vision, slow, to_atuple
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
						import tensorflow as tf

						from transformers.models.swin.modeling_tf_swin import (
						    TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFSwinForImageClassification,
						    TFSwinForMaskedImageModeling,
						    TFSwinModel,
						)


if is_vision_available():
						from PIL import Image

						from transformers import AutoImageProcessor






class 				__magic_name__  :







	"""simple docstring"""



	def __init__(				self	,   _lowercase	,   _lowercase=13	,   _lowercase=32	,   _lowercase=2	,   _lowercase=3	,   _lowercase=16	,   _lowercase=[1, 2, 1]	,   _lowercase=[2, 2, 4]	,   _lowercase=2	,   _lowercase=2.0	,   _lowercase=True	,   _lowercase=0.0	,   _lowercase=0.0	,   _lowercase=0.1	,   _lowercase="gelu"	,   _lowercase=False	,   _lowercase=True	,   _lowercase=0.02	,   _lowercase=1E-5	,   _lowercase=True	,   _lowercase=None	,   _lowercase=True	,   _lowercase=10	,   _lowercase=8	,   )    ->	None:
							lowercase_   : Dict   	=      parent
							lowercase_   : Any   	=      batch_size
							lowercase_   : Dict   	=      image_size
							lowercase_   : Tuple   	=      patch_size
							lowercase_   : Dict   	=      num_channels
							lowercase_   : str   	=      embed_dim
							lowercase_   : List[Any]   	=      depths
							lowercase_   : Optional[Any]   	=      num_heads
							lowercase_   : Tuple   	=      window_size
							lowercase_   : Any   	=      mlp_ratio
							lowercase_   : Dict   	=      qkv_bias
							lowercase_   : Union[str, Any]   	=      hidden_dropout_prob
							lowercase_   : Union[str, Any]   	=      attention_probs_dropout_prob
							lowercase_   : str   	=      drop_path_rate
							lowercase_   : List[str]   	=      hidden_act
							lowercase_   : Union[str, Any]   	=      use_absolute_embeddings
							lowercase_   : Union[str, Any]   	=      patch_norm
							lowercase_   : List[Any]   	=      layer_norm_eps
							lowercase_   : Optional[Any]   	=      initializer_range
							lowercase_   : List[str]   	=      is_training
							lowercase_   : Dict   	=      scope
							lowercase_   : List[Any]   	=      use_labels
							lowercase_   : Dict   	=      type_sequence_label_size
							lowercase_   : Union[str, Any]   	=      encoder_stride
	def 							lowerCamelCase__		(				self							)    ->	List[str]:
							lowercase_   : Dict   	=      floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]							)

							lowercase_   : Any   	=      None
							if self.use_labels:
													lowercase_   : Tuple   	=      ids_tensor([self.batch_size]	,   self.type_sequence_label_size							)

							lowercase_   : Union[str, Any]   	=      self.get_config()

							return config, pixel_values, labels
	def 							lowerCamelCase__		(				self							)    ->	Optional[int]:
							return SwinConfig(
							    image_size=self.image_size	,   patch_size=self.patch_size	,   num_channels=self.num_channels	,   embed_dim=self.embed_dim	,   depths=self.depths	,   num_heads=self.num_heads	,   window_size=self.window_size	,   mlp_ratio=self.mlp_ratio	,   qkv_bias=self.qkv_bias	,   hidden_dropout_prob=self.hidden_dropout_prob	,   attention_probs_dropout_prob=self.attention_probs_dropout_prob	,   drop_path_rate=self.drop_path_rate	,   hidden_act=self.hidden_act	,   use_absolute_embeddings=self.use_absolute_embeddings	,   path_norm=self.patch_norm	,   layer_norm_eps=self.layer_norm_eps	,   initializer_range=self.initializer_range	,   encoder_stride=self.encoder_stride	,   )
	def 							lowerCamelCase__		(				self	,   _lowercase	,   _lowercase	,   _lowercase							)    ->	Tuple:
							lowercase_   : Union[str, Any]   	=      TFSwinModel(config=_lowercase							)
							lowercase_   : Any   	=      model(_lowercase							)

							lowercase_   : List[str]   	=      ((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths							) - 1))
							lowercase_   : Optional[Any]   	=      int(config.embed_dim * 2 ** (len(config.depths							) - 1)							)

							self.parent.assertEqual(result.last_hidden_state.shape	,   (self.batch_size, expected_seq_len, expected_dim)							)
	def 							lowerCamelCase__		(				self	,   _lowercase	,   _lowercase	,   _lowercase							)    ->	Tuple:
							lowercase_   : Optional[int]   	=      TFSwinForMaskedImageModeling(config=_lowercase							)
							lowercase_   : int   	=      model(_lowercase							)
							self.parent.assertEqual(
							    result.logits.shape	,   (self.batch_size, self.num_channels, self.image_size, self.image_size)							)

							# test greyscale images
							lowercase_   : List[Any]   	=      1
							lowercase_   : Any   	=      TFSwinForMaskedImageModeling(_lowercase							)

							lowercase_   : List[Any]   	=      floats_tensor([self.batch_size, 1, self.image_size, self.image_size]							)
							lowercase_   : Any   	=      model(_lowercase							)
							self.parent.assertEqual(result.logits.shape	,   (self.batch_size, 1, self.image_size, self.image_size)							)
	def 							lowerCamelCase__		(				self	,   _lowercase	,   _lowercase	,   _lowercase							)    ->	List[str]:
							lowercase_   : int   	=      self.type_sequence_label_size
							lowercase_   : Dict   	=      TFSwinForImageClassification(_lowercase							)
							lowercase_   : Optional[Any]   	=      model(_lowercase	,   labels=_lowercase							)
							self.parent.assertEqual(result.logits.shape	,   (self.batch_size, self.type_sequence_label_size)							)

							# test greyscale images
							lowercase_   : List[Any]   	=      1
							lowercase_   : List[Any]   	=      TFSwinForImageClassification(_lowercase							)
							lowercase_   : Optional[Any]   	=      floats_tensor([self.batch_size, 1, self.image_size, self.image_size]							)
							lowercase_   : Tuple   	=      model(_lowercase							)
							self.parent.assertEqual(result.logits.shape	,   (self.batch_size, self.type_sequence_label_size)							)







	def 							lowerCamelCase__		(				self							)    ->	Union[str, Any]:
							lowercase_   : Any   	=      self.prepare_config_and_inputs()
							lowercase_	,   lowercase_	,   lowercase_   : Union[str, Any]   	=      config_and_inputs
							lowercase_   : Dict   	=      {'pixel_values': pixel_values}
							return config, inputs_dict






@require_tf
class 				__magic_name__  (						UpperCAmelCase_,    UpperCAmelCase_,    unittest.TestCase					):







	"""simple docstring"""



	SCREAMING_SNAKE_CASE_       :   Any   =	(
	    (
	        TFSwinModel,
	        TFSwinForImageClassification,
	        TFSwinForMaskedImageModeling,
	    )
	    if is_tf_available()
	    else ()
	)
	SCREAMING_SNAKE_CASE_       :   Tuple   =	(
	    {'feature-extraction': TFSwinModel, 'image-classification': TFSwinForImageClassification}
	    if is_tf_available()
	    else {}
	)

	SCREAMING_SNAKE_CASE_       :   Dict   =	False
	SCREAMING_SNAKE_CASE_       :   Optional[Any]   =	False
	SCREAMING_SNAKE_CASE_       :   str   =	False
	SCREAMING_SNAKE_CASE_       :   int   =	False
	def 							lowerCamelCase__		(				self							)    ->	Any:
							lowercase_   : List[Any]   	=      TFSwinModelTester(self							)
							lowercase_   : Union[str, Any]   	=      ConfigTester(self	,   config_class=_lowercase	,   embed_dim=37							)
	def 							lowerCamelCase__		(				self							)    ->	int:
							self.create_and_test_config_common_properties()
							self.config_tester.create_and_test_config_to_json_string()
							self.config_tester.create_and_test_config_to_json_file()
							self.config_tester.create_and_test_config_from_and_save_pretrained()
							self.config_tester.create_and_test_config_with_num_labels()
							self.config_tester.check_config_can_be_init_without_params()
							self.config_tester.check_config_arguments_init()
	def 							lowerCamelCase__		(				self							)    ->	Optional[int]:
							return
	def 							lowerCamelCase__		(				self							)    ->	Optional[int]:
							lowercase_   : Optional[Any]   	=      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*_lowercase							)
	def 							lowerCamelCase__		(				self							)    ->	Optional[Any]:
							lowercase_   : Any   	=      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_image_modeling(*_lowercase							)
	def 							lowerCamelCase__		(				self							)    ->	Tuple:
							lowercase_   : Tuple   	=      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_image_classification(*_lowercase							)
	@unittest.skip(reason='Swin does not use inputs_embeds'							)
	def 							lowerCamelCase__		(				self							)    ->	Optional[Any]:
							pass
	def 							lowerCamelCase__		(				self							)    ->	str:
							lowercase_	,   lowercase_   : Union[str, Any]   	=      self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes:
													lowercase_   : Optional[int]   	=      model_class(_lowercase							)
													self.assertIsInstance(model.get_input_embeddings()	,   tf.keras.layers.Layer							)
													lowercase_   : List[str]   	=      model.get_output_embeddings()
													self.assertTrue(x is None or isinstance(_lowercase	,   tf.keras.layers.Dense							)							)
	def 							lowerCamelCase__		(				self							)    ->	List[str]:
							lowercase_	,   lowercase_   : int   	=      self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes:
													lowercase_   : Dict   	=      model_class(_lowercase							)
													lowercase_   : Dict   	=      inspect.signature(model.call							)
													# signature.parameters is an OrderedDict => so arg_names order is deterministic
													lowercase_   : List[Any]   	=      [*signature.parameters.keys()]

													lowercase_   : Any   	=      ['pixel_values']
													self.assertListEqual(arg_names[:1]	,   _lowercase							)
	def 							lowerCamelCase__		(				self							)    ->	Dict:
							lowercase_	,   lowercase_   : int   	=      self.model_tester.prepare_config_and_inputs_for_common()
							lowercase_   : Any   	=      True

							for model_class in self.all_model_classes:
													lowercase_   : str   	=      True
													lowercase_   : Dict   	=      False
													lowercase_   : Tuple   	=      True
													lowercase_   : List[str]   	=      model_class(_lowercase							)
													lowercase_   : int   	=      model(**self._prepare_for_class(_lowercase	,   _lowercase							)							)
													lowercase_   : str   	=      outputs.attentions
													lowercase_   : Optional[Any]   	=      len(self.model_tester.depths							)
													self.assertEqual(len(_lowercase							)	,   _lowercase							)

													# check that output_attentions also work using config
													del inputs_dict["output_attentions"]
													lowercase_   : int   	=      True
													lowercase_   : Tuple   	=      config.window_size**2
													lowercase_   : List[str]   	=      model_class(_lowercase							)
													lowercase_   : int   	=      model(**self._prepare_for_class(_lowercase	,   _lowercase							)							)
													lowercase_   : Optional[int]   	=      outputs.attentions
													self.assertEqual(len(_lowercase							)	,   _lowercase							)

													self.assertListEqual(
													    list(attentions[0].shape[-3:]							)	,   [self.model_tester.num_heads[0], window_size_squared, window_size_squared]	,   )
													lowercase_   : Dict   	=      len(_lowercase							)

													# Check attention is always last and order is fine
													lowercase_   : Any   	=      True
													lowercase_   : int   	=      True
													lowercase_   : Union[str, Any]   	=      model_class(_lowercase							)
													lowercase_   : Dict   	=      model(**self._prepare_for_class(_lowercase	,   _lowercase							)							)

													if hasattr(self.model_tester	,   'num_hidden_states_types'							):
																			lowercase_   : List[Any]   	=      self.model_tester.num_hidden_states_types
													else:
																			# also another +1 for reshaped_hidden_states
																			lowercase_   : Tuple   	=      2
													self.assertEqual(out_len + added_hidden_states	,   len(_lowercase							)							)

													lowercase_   : int   	=      outputs.attentions

													self.assertEqual(len(_lowercase							)	,   _lowercase							)

													self.assertListEqual(
													    list(self_attentions[0].shape[-3:]							)	,   [self.model_tester.num_heads[0], window_size_squared, window_size_squared]	,   )
	def 							lowerCamelCase__		(				self	,   _lowercase	,   _lowercase	,   _lowercase	,   _lowercase							)    ->	Optional[int]:
							lowercase_   : Union[str, Any]   	=      model_class(_lowercase							)
							lowercase_   : str   	=      model(**self._prepare_for_class(_lowercase	,   _lowercase							)							)
							lowercase_   : List[str]   	=      outputs.hidden_states

							lowercase_   : Union[str, Any]   	=      getattr(
							    self.model_tester	,   'expected_num_hidden_layers'	,   len(self.model_tester.depths							) + 1							)
							self.assertEqual(len(_lowercase							)	,   _lowercase							)

							# Swin has a different seq_length
							lowercase_   : Optional[int]   	=      to_atuple(config.patch_size							)

							lowercase_   : Optional[int]   	=      (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

							self.assertListEqual(
							    list(hidden_states[0].shape[-2:]							)	,   [num_patches, self.model_tester.embed_dim]	,   )

							lowercase_   : List[Any]   	=      outputs.reshaped_hidden_states
							self.assertEqual(len(_lowercase							)	,   _lowercase							)

							lowercase_	,   lowercase_	,   lowercase_	,   lowercase_   : Any   	=      reshaped_hidden_states[0].shape

							lowercase_   : List[Any]   	=      tf.reshape(reshaped_hidden_states[0]	,   (batch_size, num_channels, height * width)							)
							lowercase_   : List[str]   	=      tf.transpose(_lowercase	,   (0, 2, 1)							)

							self.assertListEqual(
							    list(reshaped_hidden_states.shape[-2:]							)	,   [num_patches, self.model_tester.embed_dim]	,   )
	def 							lowerCamelCase__		(				self							)    ->	Dict:
							lowercase_	,   lowercase_   : Tuple   	=      self.model_tester.prepare_config_and_inputs_for_common()

							lowercase_   : List[Any]   	=      to_atuple(self.model_tester.image_size							)

							for model_class in self.all_model_classes:
													lowercase_   : Optional[Any]   	=      True
													self.check_hidden_states_output(_lowercase	,   _lowercase	,   _lowercase	,   _lowercase							)

													# check that output_hidden_states also work using config
													del inputs_dict["output_hidden_states"]
													lowercase_   : Any   	=      True

													self.check_hidden_states_output(_lowercase	,   _lowercase	,   _lowercase	,   _lowercase							)
	def 							lowerCamelCase__		(				self							)    ->	str:
							lowercase_	,   lowercase_   : str   	=      self.model_tester.prepare_config_and_inputs_for_common()
							lowercase_   : Union[str, Any]   	=      3

							lowercase_   : List[Any]   	=      to_atuple(self.model_tester.image_size							)
							lowercase_   : Union[str, Any]   	=      to_atuple(config.patch_size							)

							lowercase_   : Tuple   	=      image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
							lowercase_   : List[Any]   	=      image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

							for model_class in self.all_model_classes:
													lowercase_   : str   	=      True
													self.check_hidden_states_output(_lowercase	,   _lowercase	,   _lowercase	,   (padded_height, padded_width)							)

													# check that output_hidden_states also work using config
													del inputs_dict["output_hidden_states"]
													lowercase_   : Optional[Any]   	=      True
													self.check_hidden_states_output(_lowercase	,   _lowercase	,   _lowercase	,   (padded_height, padded_width)							)







	@slow
	def 							lowerCamelCase__		(				self							)    ->	List[Any]:
							for model_name in TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
													lowercase_   : List[Any]   	=      TFSwinModel.from_pretrained(_lowercase							)
													self.assertIsNotNone(_lowercase							)






@require_vision
@require_tf
class 				__magic_name__  (						unittest.TestCase					):







	"""simple docstring"""



	@cached_property
	def 							lowerCamelCase__		(				self							)    ->	Tuple:
							return (
							    AutoImageProcessor.from_pretrained('microsoft/swin-tiny-patch4-window7-224'							)
							    if is_vision_available()
							    else None
							)







	@slow
	def 							lowerCamelCase__		(				self							)    ->	str:
							lowercase_   : Optional[Any]   	=      TFSwinForImageClassification.from_pretrained('microsoft/swin-tiny-patch4-window7-224'							)
							lowercase_   : Any   	=      self.default_image_processor

							lowercase_   : List[str]   	=      Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'							)
							lowercase_   : int   	=      image_processor(images=_lowercase	,   return_tensors='tf'							)

							# forward pass
							lowercase_   : List[str]   	=      model(_lowercase							)

							# verify the logits
							lowercase_   : List[str]   	=      tf.TensorShape((1, 1000)							)
							self.assertEqual(outputs.logits.shape	,   _lowercase							)
							lowercase_   : Any   	=      tf.constant([-0.09_48, -0.64_54, -0.09_21]							)
							self.assertTrue(np.allclose(outputs.logits[0, :3]	,   _lowercase	,   atol=1E-4							)							)








'''simple docstring'''

import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
						from tokenizers import Tokenizer
						from tokenizers.models import WordPiece






class 				__magic_name__  (						unittest.TestCase					):







	"""simple docstring"""



	def 							lowerCamelCase__		(				self	,   _lowercase							)    ->	Optional[int]:
							lowercase_   : Optional[int]   	=      list(tokenizer_class.max_model_input_sizes.keys()							)
							for model_name in sa_models[:1]:
													lowercase_   : List[str]   	=      tokenizer_class.from_pretrained(_lowercase							)
													self.assertIsNotNone(_lowercase							)
													self.assertIsInstance(_lowercase	,   _lowercase							)
													self.assertIsInstance(_lowercase	,   _lowercase							)

													for special_tok in tokenizer.all_special_tokens:
																			self.assertIsInstance(_lowercase	,   _lowercase							)
																			lowercase_   : Optional[int]   	=      tokenizer.convert_tokens_to_ids(_lowercase							)
																			self.assertIsInstance(_lowercase	,   _lowercase							)
	def 							lowerCamelCase__		(				self	,   _lowercase	,   _lowercase = None							)    ->	str:
							lowercase_   : int   	=      pickle.dumps(_lowercase							)
							self.assertIsNotNone(_lowercase							)

							lowercase_   : Any   	=      pickle.loads(_lowercase							)

							# Ensure is_fast is correctly restored
							self.assertEqual(be_restored.is_fast	,   be_original.is_fast							)

							# Ensure encodings are potentially correctly restored
							if be_original.is_fast:
													self.assertIsNotNone(be_restored.encodings							)
							else:
													self.assertIsNone(be_restored.encodings							)

							# Ensure the keys are the same
							for original_v, restored_v in zip(be_original.values()	,   be_restored.values()							):
													if equal_op:
																			self.assertTrue(equal_op(_lowercase	,   _lowercase							)							)
													else:
																			self.assertEqual(_lowercase	,   _lowercase							)
	@slow
	def 							lowerCamelCase__		(				self							)    ->	List[str]:
							self.check_tokenizer_from_pretrained(_lowercase							)
	def 							lowerCamelCase__		(				self							)    ->	str:
							self.assertEqual(TensorType('tf'							)	,   TensorType.TENSORFLOW							)
							self.assertEqual(TensorType('pt'							)	,   TensorType.PYTORCH							)
							self.assertEqual(TensorType('np'							)	,   TensorType.NUMPY							)
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	Dict:
							import numpy as np

							lowercase_   : Any   	=      BertTokenizer.from_pretrained('bert-base-cased'							)
							lowercase_   : str   	=      BertTokenizerFast.from_pretrained('bert-base-cased'							)

							# Python no tensor
							with self.subTest('BatchEncoding (Python, return_tensors=None)'							):
													self.assert_dump_and_restore(tokenizer_p('Small example to encode'							)							)

							with self.subTest('BatchEncoding (Python, return_tensors=NUMPY)'							):
													self.assert_dump_and_restore(
													    tokenizer_p('Small example to encode'	,   return_tensors=TensorType.NUMPY							)	,   np.array_equal							)

							with self.subTest('BatchEncoding (Rust, return_tensors=None)'							):
													self.assert_dump_and_restore(tokenizer_r('Small example to encode'							)							)

							with self.subTest('BatchEncoding (Rust, return_tensors=NUMPY)'							):
													self.assert_dump_and_restore(
													    tokenizer_r('Small example to encode'	,   return_tensors=TensorType.NUMPY							)	,   np.array_equal							)
	@require_tf
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	Optional[int]:
							import tensorflow as tf

							def tf_array_equals(_lowercase	,   _lowercase							):
													return tf.reduce_all(tf.equal(_lowercase	,   _lowercase							)							)

							lowercase_   : Optional[Any]   	=      BertTokenizer.from_pretrained('bert-base-cased'							)
							lowercase_   : str   	=      BertTokenizerFast.from_pretrained('bert-base-cased'							)

							with self.subTest('BatchEncoding (Python, return_tensors=TENSORFLOW)'							):
													self.assert_dump_and_restore(
													    tokenizer_p('Small example to encode'	,   return_tensors=TensorType.TENSORFLOW							)	,   _lowercase							)

							with self.subTest('BatchEncoding (Rust, return_tensors=TENSORFLOW)'							):
													self.assert_dump_and_restore(
													    tokenizer_r('Small example to encode'	,   return_tensors=TensorType.TENSORFLOW							)	,   _lowercase							)
	@require_torch
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	List[str]:
							import torch

							lowercase_   : Dict   	=      BertTokenizer.from_pretrained('bert-base-cased'							)
							lowercase_   : Optional[int]   	=      BertTokenizerFast.from_pretrained('bert-base-cased'							)

							with self.subTest('BatchEncoding (Python, return_tensors=PYTORCH)'							):
													self.assert_dump_and_restore(
													    tokenizer_p('Small example to encode'	,   return_tensors=TensorType.PYTORCH							)	,   torch.equal							)

							with self.subTest('BatchEncoding (Rust, return_tensors=PYTORCH)'							):
													self.assert_dump_and_restore(
													    tokenizer_r('Small example to encode'	,   return_tensors=TensorType.PYTORCH							)	,   torch.equal							)
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	Tuple:
							lowercase_   : Dict   	=      BertTokenizer.from_pretrained('bert-base-cased'							)
							lowercase_   : Tuple   	=      BertTokenizerFast.from_pretrained('bert-base-cased'							)

							with self.subTest('Python Tokenizer'							):
													self.assertFalse(tokenizer_p('Small example to_encode'							).is_fast							)

							with self.subTest('Rust Tokenizer'							):
													self.assertTrue(tokenizer_r('Small example to_encode'							).is_fast							)
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	List[Any]:
							lowercase_   : Optional[Any]   	=      BertTokenizerFast.from_pretrained('bert-base-cased'							)
							lowercase_   : Dict   	=      tokenizer_r(['Test', '\xad', 'test']	,   is_split_into_words=_lowercase							)

							self.assertEqual(encoded.word_to_tokens(0							)	,   TokenSpan(start=1	,   end=2							)							)
							self.assertEqual(encoded.word_to_tokens(1							)	,   _lowercase							)
							self.assertEqual(encoded.word_to_tokens(2							)	,   TokenSpan(start=2	,   end=3							)							)
	def 							lowerCamelCase__		(				self							)    ->	str:
							lowercase_   : Dict   	=      BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}							)
							lowercase_   : Any   	=      batch.convert_to_tensors(tensor_type='np'							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (2, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (2,)							)
							# test converting the converted
							with CaptureStderr() as cs:
													lowercase_   : Dict   	=      batch.convert_to_tensors(tensor_type='np'							)
							self.assertFalse(len(cs.err							)	,   msg=f"should have no warning, but got {cs.err}"							)

							lowercase_   : Union[str, Any]   	=      BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}							)
							lowercase_   : Optional[int]   	=      batch.convert_to_tensors(tensor_type='np'	,   prepend_batch_axis=_lowercase							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (1, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (1,)							)
	@require_torch
	def 							lowerCamelCase__		(				self							)    ->	Union[str, Any]:
							lowercase_   : Dict   	=      BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}							)
							lowercase_   : Optional[Any]   	=      batch.convert_to_tensors(tensor_type='pt'							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (2, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (2,)							)
							# test converting the converted
							with CaptureStderr() as cs:
													lowercase_   : Optional[Any]   	=      batch.convert_to_tensors(tensor_type='pt'							)
							self.assertFalse(len(cs.err							)	,   msg=f"should have no warning, but got {cs.err}"							)

							lowercase_   : int   	=      BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}							)
							lowercase_   : Union[str, Any]   	=      batch.convert_to_tensors(tensor_type='pt'	,   prepend_batch_axis=_lowercase							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (1, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (1,)							)
	@require_tf
	def 							lowerCamelCase__		(				self							)    ->	Tuple:
							lowercase_   : Dict   	=      BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}							)
							lowercase_   : List[Any]   	=      batch.convert_to_tensors(tensor_type='tf'							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (2, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (2,)							)
							# test converting the converted
							with CaptureStderr() as cs:
													lowercase_   : Tuple   	=      batch.convert_to_tensors(tensor_type='tf'							)
							self.assertFalse(len(cs.err							)	,   msg=f"should have no warning, but got {cs.err}"							)

							lowercase_   : int   	=      BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}							)
							lowercase_   : Optional[Any]   	=      batch.convert_to_tensors(tensor_type='tf'	,   prepend_batch_axis=_lowercase							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (1, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (1,)							)
	@require_flax
	def 							lowerCamelCase__		(				self							)    ->	Union[str, Any]:
							lowercase_   : int   	=      BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}							)
							lowercase_   : Optional[Any]   	=      batch.convert_to_tensors(tensor_type='jax'							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (2, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (2,)							)
							# test converting the converted
							with CaptureStderr() as cs:
													lowercase_   : Dict   	=      batch.convert_to_tensors(tensor_type='jax'							)
							self.assertFalse(len(cs.err							)	,   msg=f"should have no warning, but got {cs.err}"							)

							lowercase_   : List[Any]   	=      BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}							)
							lowercase_   : Dict   	=      batch.convert_to_tensors(tensor_type='jax'	,   prepend_batch_axis=_lowercase							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (1, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (1,)							)
	def 							lowerCamelCase__		(				self							)    ->	List[str]:
							lowercase_   : Any   	=      [{'input_ids': np.array([0, 1, 2]							)}, {'input_ids': np.array([0, 1, 2, 3]							)}]
							lowercase_   : str   	=      BertTokenizer.from_pretrained('bert-base-cased'							)

							lowercase_   : Any   	=      tokenizer.pad(_lowercase	,   padding=_lowercase							)
							self.assertTrue(isinstance(batch['input_ids']	,   np.ndarray							)							)
							self.assertEqual(batch['input_ids'].tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
							lowercase_   : List[str]   	=      tokenizer.pad(_lowercase	,   padding=_lowercase	,   return_tensors='np'							)
							self.assertTrue(isinstance(batch['input_ids']	,   np.ndarray							)							)
							self.assertEqual(batch['input_ids'].tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
	@require_torch
	def 							lowerCamelCase__		(				self							)    ->	List[Any]:
							import torch

							lowercase_   : Union[str, Any]   	=      [{'input_ids': torch.tensor([0, 1, 2]							)}, {'input_ids': torch.tensor([0, 1, 2, 3]							)}]
							lowercase_   : Optional[int]   	=      BertTokenizer.from_pretrained('bert-base-cased'							)

							lowercase_   : str   	=      tokenizer.pad(_lowercase	,   padding=_lowercase							)
							self.assertTrue(isinstance(batch['input_ids']	,   torch.Tensor							)							)
							self.assertEqual(batch['input_ids'].tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
							lowercase_   : Tuple   	=      tokenizer.pad(_lowercase	,   padding=_lowercase	,   return_tensors='pt'							)
							self.assertTrue(isinstance(batch['input_ids']	,   torch.Tensor							)							)
							self.assertEqual(batch['input_ids'].tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
	@require_tf
	def 							lowerCamelCase__		(				self							)    ->	Any:
							import tensorflow as tf

							lowercase_   : Any   	=      [{'input_ids': tf.constant([0, 1, 2]							)}, {'input_ids': tf.constant([0, 1, 2, 3]							)}]
							lowercase_   : Dict   	=      BertTokenizer.from_pretrained('bert-base-cased'							)

							lowercase_   : Any   	=      tokenizer.pad(_lowercase	,   padding=_lowercase							)
							self.assertTrue(isinstance(batch['input_ids']	,   tf.Tensor							)							)
							self.assertEqual(batch['input_ids'].numpy().tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
							lowercase_   : Dict   	=      tokenizer.pad(_lowercase	,   padding=_lowercase	,   return_tensors='tf'							)
							self.assertTrue(isinstance(batch['input_ids']	,   tf.Tensor							)							)
							self.assertEqual(batch['input_ids'].numpy().tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	str:
							lowercase_   : Optional[int]   	=      Tokenizer(WordPiece(unk_token='[UNK]'							)							)
							PreTrainedTokenizerFast(tokenizer_object=_lowercase							)







	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	int:
							lowercase_   : Union[str, Any]   	=      Tokenizer(WordPiece(unk_token='[UNK]'							)							)
							with tempfile.TemporaryDirectory() as tmpdirname:
													bert_tokenizer.save(os.path.join(_lowercase	,   'tokenizer.json'							)							)
													PreTrainedTokenizerFast(tokenizer_file=os.path.join(_lowercase	,   'tokenizer.json'							)							)
