



from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


UpperCAmelCase_			      =  logging.get_logger(__name__)

UpperCAmelCase_			      =  {
    '''google/bigbird-pegasus-large-arxiv''': (
        '''https://huggingface.co/google/bigbird-pegasus-large-arxiv/resolve/main/config.json'''
    ),
    '''google/bigbird-pegasus-large-pubmed''': (
        '''https://huggingface.co/google/bigbird-pegasus-large-pubmed/resolve/main/config.json'''
    ),
    '''google/bigbird-pegasus-large-bigpatent''': (
        '''https://huggingface.co/google/bigbird-pegasus-large-bigpatent/resolve/main/config.json'''
    ),
    # See all BigBirdPegasus models at https://huggingface.co/models?filter=bigbird_pegasus
}





class 		__SCREAMING_SNAKE_CASE      (				lowercase__      ):






				"""simple docstring"""

				SCREAMING_SNAKE_CASE_ 				=  'bigbird_pegasus'
				SCREAMING_SNAKE_CASE_ 				=  ['past_key_values']
				SCREAMING_SNAKE_CASE_ 				=  {
				    'num_attention_heads': 'encoder_attention_heads',
				    'hidden_size': 'd_model',
				    'attention_probs_dropout_prob': 'attention_dropout',
				}







				def __init__(				self						,	SCREAMING_SNAKE_CASE__=9_61_03						,	SCREAMING_SNAKE_CASE__=40_96						,	SCREAMING_SNAKE_CASE__=16						,	SCREAMING_SNAKE_CASE__=40_96						,	SCREAMING_SNAKE_CASE__=16						,	SCREAMING_SNAKE_CASE__=16						,	SCREAMING_SNAKE_CASE__=40_96						,	SCREAMING_SNAKE_CASE__=16						,	SCREAMING_SNAKE_CASE__=0.0						,	SCREAMING_SNAKE_CASE__=0.0						,	SCREAMING_SNAKE_CASE__=True						,	SCREAMING_SNAKE_CASE__=True						,	SCREAMING_SNAKE_CASE__="gelu_new"						,	SCREAMING_SNAKE_CASE__=10_24						,	SCREAMING_SNAKE_CASE__=0.1						,	SCREAMING_SNAKE_CASE__=0.0						,	SCREAMING_SNAKE_CASE__=0.0						,	SCREAMING_SNAKE_CASE__=0.02						,	SCREAMING_SNAKE_CASE__=2						,	SCREAMING_SNAKE_CASE__=0.0						,	SCREAMING_SNAKE_CASE__=True						,	SCREAMING_SNAKE_CASE__=0						,	SCREAMING_SNAKE_CASE__=2						,	SCREAMING_SNAKE_CASE__=1						,	SCREAMING_SNAKE_CASE__="block_sparse"						,	SCREAMING_SNAKE_CASE__=64						,	SCREAMING_SNAKE_CASE__=3						,	SCREAMING_SNAKE_CASE__=False						,	**SCREAMING_SNAKE_CASE__						,	):






					"""simple docstring"""

					_snake_case					:							Dict					     =   vocab_size
					_snake_case					:							Any					     =   max_position_embeddings
					_snake_case					:							Any					     =   d_model
					_snake_case					:							Dict					     =   encoder_ffn_dim
					_snake_case					:							Union[str, Any]					     =   encoder_layers
					_snake_case					:							Optional[int]					     =   encoder_attention_heads
					_snake_case					:							Optional[int]					     =   decoder_ffn_dim
					_snake_case					:							List[Any]					     =   decoder_layers
					_snake_case					:							str					     =   decoder_attention_heads
					_snake_case					:							Optional[Any]					     =   dropout
					_snake_case					:							Optional[Any]					     =   attention_dropout
					_snake_case					:							int					     =   activation_dropout
					_snake_case					:							Any					     =   activation_function
					_snake_case					:							Optional[Any]					     =   init_std
					_snake_case					:							Tuple					     =   encoder_layerdrop
					_snake_case					:							List[Any]					     =   decoder_layerdrop
					_snake_case					:							Optional[Any]					     =   classifier_dropout
					_snake_case					:							List[Any]					     =   use_cache
					_snake_case					:							Union[str, Any]					     =   encoder_layers
					_snake_case					:							Union[str, Any]					     =   scale_embedding  # scale factor will be sqrt(d_model) if True

					# extra config
					_snake_case					:							str					     =   attention_type
					_snake_case					:							Tuple					     =   block_size
					_snake_case					:							List[Any]					     =   num_random_blocks
					_snake_case					:							List[str]					     =   use_bias

					super().__init__(
					    pad_token_id=SCREAMING_SNAKE_CASE__						,	bos_token_id=SCREAMING_SNAKE_CASE__						,	eos_token_id=SCREAMING_SNAKE_CASE__						,	is_encoder_decoder=SCREAMING_SNAKE_CASE__						,	decoder_start_token_id=SCREAMING_SNAKE_CASE__						,	**SCREAMING_SNAKE_CASE__						,	)





class 		__SCREAMING_SNAKE_CASE      (				lowercase__      ):






				"""simple docstring"""

				@property
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					if self.task in ["default", "seq2seq-lm"]:
						_snake_case					:							Dict					     =   OrderedDict(
						    [
						        ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
						        ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
						    ]      )

						if self.use_past:
							_snake_case					:							Any					     =   {0: """batch"""}
							_snake_case					:							str					     =   {0: """batch""", 1: """past_decoder_sequence + sequence"""}
						else:
							_snake_case					:							int					     =   {0: """batch""", 1: """decoder_sequence"""}
							_snake_case					:							Optional[Any]					     =   {0: """batch""", 1: """decoder_sequence"""}

						if self.use_past:
							self.fill_with_past_key_values_(SCREAMING_SNAKE_CASE__						,	direction="""inputs"""      )
					elif self.task == "causal-lm":
						# TODO: figure this case out.
						_snake_case					:							Any					     =   OrderedDict(
						    [
						        ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
						        ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
						    ]      )
						if self.use_past:
							_snake_case     , _snake_case					:							Tuple					     =   self.num_layers
							for i in range(SCREAMING_SNAKE_CASE__      ):
								_snake_case					:							Dict					     =   {0: """batch""", 2: """past_sequence + sequence"""}
								_snake_case					:							int					     =   {0: """batch""", 2: """past_sequence + sequence"""}
					else:
						_snake_case					:							Dict					     =   OrderedDict(
						    [
						        ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
						        ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
						        ("""decoder_input_ids""", {0: """batch""", 1: """decoder_sequence"""}),
						        ("""decoder_attention_mask""", {0: """batch""", 1: """decoder_sequence"""}),
						    ]      )

					return common_inputs







				@property
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					if self.task in ["default", "seq2seq-lm"]:
						_snake_case					:							str					     =   super().outputs
					else:
						_snake_case					:							List[Any]					     =   super(SCREAMING_SNAKE_CASE__						,	self      ).outputs
						if self.use_past:
							_snake_case     , _snake_case					:							List[str]					     =   self.num_layers
							for i in range(SCREAMING_SNAKE_CASE__      ):
								_snake_case					:							Optional[Any]					     =   {0: """batch""", 2: """past_sequence + sequence"""}
								_snake_case					:							List[str]					     =   {0: """batch""", 2: """past_sequence + sequence"""}
					return common_outputs







				def 		__lowerCamelCase(				self						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__ = -1						,	SCREAMING_SNAKE_CASE__ = -1						,	SCREAMING_SNAKE_CASE__ = False						,	SCREAMING_SNAKE_CASE__ = None						,	):






					"""simple docstring"""

					_snake_case					:							Optional[Any]					     =   self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
					    SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )

					# Generate decoder inputs
					_snake_case					:							Union[str, Any]					     =   seq_length if not self.use_past else 1
					_snake_case					:							Tuple					     =   self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
					    SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )
					_snake_case					:							Union[str, Any]					     =   {f'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
					_snake_case					:							Optional[int]					     =   dict(**SCREAMING_SNAKE_CASE__						,	**SCREAMING_SNAKE_CASE__      )

					if self.use_past:
						if not is_torch_available():
							raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""      )
						else:
							import torch
						_snake_case     , _snake_case					:							int					     =   common_inputs["""input_ids"""].shape
						_snake_case					:							List[Any]					     =   common_inputs["""decoder_input_ids"""].shape[1]
						_snake_case     , _snake_case					:							List[Any]					     =   self.num_attention_heads
						_snake_case					:							Any					     =   (
						    batch,
						    num_encoder_attention_heads,
						    encoder_seq_length,
						    self._config.hidden_size // num_encoder_attention_heads,
						)
						_snake_case					:							int					     =   decoder_seq_length + 3
						_snake_case					:							List[Any]					     =   (
						    batch,
						    num_decoder_attention_heads,
						    decoder_past_length,
						    self._config.hidden_size // num_decoder_attention_heads,
						)

						_snake_case					:							List[str]					     =   torch.cat(
						    [common_inputs["""decoder_attention_mask"""], torch.ones(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )]						,	dim=1      )

						_snake_case					:							Dict					     =   []
						# If the number of encoder and decoder layers are present in the model configuration, both are considered
						_snake_case     , _snake_case					:							str					     =   self.num_layers
						_snake_case					:							Any					     =   min(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )
						_snake_case					:							Any					     =   max(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      ) - min_num_layers
						_snake_case					:							Union[str, Any]					     =   """encoder""" if num_encoder_layers > num_decoder_layers else """decoder"""

						for _ in range(SCREAMING_SNAKE_CASE__      ):
							common_inputs["past_key_values"].append(
							    (
							        torch.zeros(SCREAMING_SNAKE_CASE__      ),
							        torch.zeros(SCREAMING_SNAKE_CASE__      ),
							        torch.zeros(SCREAMING_SNAKE_CASE__      ),
							        torch.zeros(SCREAMING_SNAKE_CASE__      ),
							    )      )
						# TODO: test this.
						_snake_case					:							List[Any]					     =   encoder_shape if remaining_side_name == """encoder""" else decoder_shape
						for _ in range(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      ):
							common_inputs["past_key_values"].append((torch.zeros(SCREAMING_SNAKE_CASE__      ), torch.zeros(SCREAMING_SNAKE_CASE__      ))      )
					return common_inputs







				def 		__lowerCamelCase(				self						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__ = -1						,	SCREAMING_SNAKE_CASE__ = -1						,	SCREAMING_SNAKE_CASE__ = False						,	SCREAMING_SNAKE_CASE__ = None						,	):






					"""simple docstring"""

					_snake_case					:							Tuple					     =   self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
					    SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )

					if self.use_past:
						if not is_torch_available():
							raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""      )
						else:
							import torch
						_snake_case     , _snake_case					:							Optional[Any]					     =   common_inputs["""input_ids"""].shape
						# Not using the same length for past_key_values
						_snake_case					:							Dict					     =   seqlen + 2
						_snake_case     , _snake_case					:							Optional[int]					     =   self.num_layers
						_snake_case     , _snake_case					:							List[Any]					     =   self.num_attention_heads
						_snake_case					:							str					     =   (
						    batch,
						    num_encoder_attention_heads,
						    past_key_values_length,
						    self._config.hidden_size // num_encoder_attention_heads,
						)

						_snake_case					:							Optional[int]					     =   common_inputs["""attention_mask"""].dtype
						_snake_case					:							str					     =   torch.cat(
						    [common_inputs["""attention_mask"""], torch.ones(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	dtype=SCREAMING_SNAKE_CASE__      )]						,	dim=1      )
						_snake_case					:							Tuple					     =   [
						    (torch.zeros(SCREAMING_SNAKE_CASE__      ), torch.zeros(SCREAMING_SNAKE_CASE__      )) for _ in range(SCREAMING_SNAKE_CASE__      )
						]
					return common_inputs







				def 		__lowerCamelCase(				self						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__ = -1						,	SCREAMING_SNAKE_CASE__ = -1						,	SCREAMING_SNAKE_CASE__ = False						,	SCREAMING_SNAKE_CASE__ = None						,	):






					"""simple docstring"""

					_snake_case					:							Optional[int]					     =   compute_effective_axis_dimension(
					    SCREAMING_SNAKE_CASE__						,	fixed_dimension=OnnxConfig.default_fixed_batch						,	num_token_to_add=0      )

					# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
					_snake_case					:							List[str]					     =   tokenizer.num_special_tokens_to_add(SCREAMING_SNAKE_CASE__      )
					_snake_case					:							Tuple					     =   compute_effective_axis_dimension(
					    SCREAMING_SNAKE_CASE__						,	fixed_dimension=OnnxConfig.default_fixed_sequence						,	num_token_to_add=SCREAMING_SNAKE_CASE__      )

					# Generate dummy inputs according to compute batch and sequence
					_snake_case					:							str					     =   [""" """.join([tokenizer.unk_token]      ) * seq_length] * batch_size
					_snake_case					:							Optional[int]					     =   dict(tokenizer(SCREAMING_SNAKE_CASE__						,	return_tensors=SCREAMING_SNAKE_CASE__      )      )
					return common_inputs







				def 		__lowerCamelCase(				self						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__ = -1						,	SCREAMING_SNAKE_CASE__ = -1						,	SCREAMING_SNAKE_CASE__ = False						,	SCREAMING_SNAKE_CASE__ = None						,	):






					"""simple docstring"""

					if self.task in ["default", "seq2seq-lm"]:
						_snake_case					:							int					     =   self._generate_dummy_inputs_for_default_and_seqaseq_lm(
						    SCREAMING_SNAKE_CASE__						,	batch_size=SCREAMING_SNAKE_CASE__						,	seq_length=SCREAMING_SNAKE_CASE__						,	is_pair=SCREAMING_SNAKE_CASE__						,	framework=SCREAMING_SNAKE_CASE__      )

					elif self.task == "causal-lm":
						_snake_case					:							List[Any]					     =   self._generate_dummy_inputs_for_causal_lm(
						    SCREAMING_SNAKE_CASE__						,	batch_size=SCREAMING_SNAKE_CASE__						,	seq_length=SCREAMING_SNAKE_CASE__						,	is_pair=SCREAMING_SNAKE_CASE__						,	framework=SCREAMING_SNAKE_CASE__      )
					else:
						_snake_case					:							Dict					     =   self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
						    SCREAMING_SNAKE_CASE__						,	batch_size=SCREAMING_SNAKE_CASE__						,	seq_length=SCREAMING_SNAKE_CASE__						,	is_pair=SCREAMING_SNAKE_CASE__						,	framework=SCREAMING_SNAKE_CASE__      )

					return common_inputs







				def 		__lowerCamelCase(				self						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      ):






					"""simple docstring"""

					if self.task in ["default", "seq2seq-lm"]:
						_snake_case					:							Union[str, Any]					     =   super()._flatten_past_key_values_(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )
					else:
						_snake_case					:							int					     =   super(SCREAMING_SNAKE_CASE__						,	self      )._flatten_past_key_values_(
						    SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )










import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
						from tokenizers import Tokenizer
						from tokenizers.models import WordPiece





class 		__SCREAMING_SNAKE_CASE      (				unittest.TestCase      ):






				"""simple docstring"""

				def 		__lowerCamelCase(				self						,	SCREAMING_SNAKE_CASE__      ):






					"""simple docstring"""

					_snake_case					:							Any					     =   list(tokenizer_class.max_model_input_sizes.keys()      )
					for model_name in sa_models[:1]:
						_snake_case					:							Optional[Any]					     =   tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__      )
						self.assertIsNotNone(SCREAMING_SNAKE_CASE__      )
						self.assertIsInstance(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )
						self.assertIsInstance(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )

						for special_tok in tokenizer.all_special_tokens:
							self.assertIsInstance(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )
							_snake_case					:							int					     =   tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__      )
							self.assertIsInstance(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )







				def 		__lowerCamelCase(				self						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__ = None      ):






					"""simple docstring"""

					_snake_case					:							int					     =   pickle.dumps(SCREAMING_SNAKE_CASE__      )
					self.assertIsNotNone(SCREAMING_SNAKE_CASE__      )

					_snake_case					:							Dict					     =   pickle.loads(SCREAMING_SNAKE_CASE__      )

					# Ensure is_fast is correctly restored
					self.assertEqual(be_restored.is_fast						,	be_original.is_fast      )

					# Ensure encodings are potentially correctly restored
					if be_original.is_fast:
						self.assertIsNotNone(be_restored.encodings      )
					else:
						self.assertIsNone(be_restored.encodings      )

					# Ensure the keys are the same
					for original_v, restored_v in zip(be_original.values()						,	be_restored.values()      ):
						if equal_op:
							self.assertTrue(equal_op(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )      )
						else:
							self.assertEqual(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )







				@slow
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					self.check_tokenizer_from_pretrained(SCREAMING_SNAKE_CASE__      )







				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					self.assertEqual(TensorType("""tf"""      )						,	TensorType.TENSORFLOW      )
					self.assertEqual(TensorType("""pt"""      )						,	TensorType.PYTORCH      )
					self.assertEqual(TensorType("""np"""      )						,	TensorType.NUMPY      )







				@require_tokenizers
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					import numpy as np

					_snake_case					:							Optional[int]					     =   BertTokenizer.from_pretrained("""bert-base-cased"""      )
					_snake_case					:							Any					     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""      )

					# Python no tensor
					with self.subTest("""BatchEncoding (Python, return_tensors=None)"""      ):
						self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""      )      )

					with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""      ):
						self.assert_dump_and_restore(
						    tokenizer_p("""Small example to encode"""						,	return_tensors=TensorType.NUMPY      )						,	np.array_equal      )

					with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""      ):
						self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""      )      )

					with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""      ):
						self.assert_dump_and_restore(
						    tokenizer_r("""Small example to encode"""						,	return_tensors=TensorType.NUMPY      )						,	np.array_equal      )







				@require_tf
				@require_tokenizers
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					import tensorflow as tf

					def tf_array_equals(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      ):
						return tf.reduce_all(tf.equal(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )      )

					_snake_case					:							int					     =   BertTokenizer.from_pretrained("""bert-base-cased"""      )
					_snake_case					:							str					     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""      )

					with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""      ):
						self.assert_dump_and_restore(
						    tokenizer_p("""Small example to encode"""						,	return_tensors=TensorType.TENSORFLOW      )						,	SCREAMING_SNAKE_CASE__      )

					with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""      ):
						self.assert_dump_and_restore(
						    tokenizer_r("""Small example to encode"""						,	return_tensors=TensorType.TENSORFLOW      )						,	SCREAMING_SNAKE_CASE__      )







				@require_torch
				@require_tokenizers
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					import torch

					_snake_case					:							Union[str, Any]					     =   BertTokenizer.from_pretrained("""bert-base-cased"""      )
					_snake_case					:							Dict					     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""      )

					with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""      ):
						self.assert_dump_and_restore(
						    tokenizer_p("""Small example to encode"""						,	return_tensors=TensorType.PYTORCH      )						,	torch.equal      )

					with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""      ):
						self.assert_dump_and_restore(
						    tokenizer_r("""Small example to encode"""						,	return_tensors=TensorType.PYTORCH      )						,	torch.equal      )







				@require_tokenizers
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							Optional[int]					     =   BertTokenizer.from_pretrained("""bert-base-cased"""      )
					_snake_case					:							int					     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""      )

					with self.subTest("""Python Tokenizer"""      ):
						self.assertFalse(tokenizer_p("""Small example to_encode"""      ).is_fast      )

					with self.subTest("""Rust Tokenizer"""      ):
						self.assertTrue(tokenizer_r("""Small example to_encode"""      ).is_fast      )







				@require_tokenizers
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							Optional[int]					     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""      )
					_snake_case					:							Dict					     =   tokenizer_r(["""Test""", """\xad""", """test"""]						,	is_split_into_words=SCREAMING_SNAKE_CASE__      )

					self.assertEqual(encoded.word_to_tokens(0      )						,	TokenSpan(start=1						,	end=2      )      )
					self.assertEqual(encoded.word_to_tokens(1      )						,	SCREAMING_SNAKE_CASE__      )
					self.assertEqual(encoded.word_to_tokens(2      )						,	TokenSpan(start=2						,	end=3      )      )







				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							List[Any]					     =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}      )
					_snake_case					:							Union[str, Any]					     =   batch.convert_to_tensors(tensor_type="""np"""      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(2, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(2,)      )
					# test converting the converted
					with CaptureStderr() as cs:
						_snake_case					:							Optional[int]					     =   batch.convert_to_tensors(tensor_type="""np"""      )
					self.assertFalse(len(cs.err      )						,	msg=f'''should have no warning, but got {cs.err}'''      )

					_snake_case					:							str					     =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}      )
					_snake_case					:							Optional[int]					     =   batch.convert_to_tensors(tensor_type="""np"""						,	prepend_batch_axis=SCREAMING_SNAKE_CASE__      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(1, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(1,)      )







				@require_torch
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							List[Any]					     =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}      )
					_snake_case					:							Union[str, Any]					     =   batch.convert_to_tensors(tensor_type="""pt"""      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(2, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(2,)      )
					# test converting the converted
					with CaptureStderr() as cs:
						_snake_case					:							Optional[Any]					     =   batch.convert_to_tensors(tensor_type="""pt"""      )
					self.assertFalse(len(cs.err      )						,	msg=f'''should have no warning, but got {cs.err}'''      )

					_snake_case					:							Union[str, Any]					     =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}      )
					_snake_case					:							Union[str, Any]					     =   batch.convert_to_tensors(tensor_type="""pt"""						,	prepend_batch_axis=SCREAMING_SNAKE_CASE__      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(1, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(1,)      )







				@require_tf
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							Optional[int]					     =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}      )
					_snake_case					:							str					     =   batch.convert_to_tensors(tensor_type="""tf"""      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(2, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(2,)      )
					# test converting the converted
					with CaptureStderr() as cs:
						_snake_case					:							int					     =   batch.convert_to_tensors(tensor_type="""tf"""      )
					self.assertFalse(len(cs.err      )						,	msg=f'''should have no warning, but got {cs.err}'''      )

					_snake_case					:							Optional[Any]					     =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}      )
					_snake_case					:							List[str]					     =   batch.convert_to_tensors(tensor_type="""tf"""						,	prepend_batch_axis=SCREAMING_SNAKE_CASE__      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(1, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(1,)      )







				@require_flax
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							List[Any]					     =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}      )
					_snake_case					:							Any					     =   batch.convert_to_tensors(tensor_type="""jax"""      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(2, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(2,)      )
					# test converting the converted
					with CaptureStderr() as cs:
						_snake_case					:							Dict					     =   batch.convert_to_tensors(tensor_type="""jax"""      )
					self.assertFalse(len(cs.err      )						,	msg=f'''should have no warning, but got {cs.err}'''      )

					_snake_case					:							List[str]					     =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}      )
					_snake_case					:							List[Any]					     =   batch.convert_to_tensors(tensor_type="""jax"""						,	prepend_batch_axis=SCREAMING_SNAKE_CASE__      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(1, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(1,)      )







				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							Any					     =   [{"""input_ids""": np.array([0, 1, 2]      )}, {"""input_ids""": np.array([0, 1, 2, 3]      )}]
					_snake_case					:							int					     =   BertTokenizer.from_pretrained("""bert-base-cased"""      )

					_snake_case					:							List[Any]					     =   tokenizer.pad(SCREAMING_SNAKE_CASE__						,	padding=SCREAMING_SNAKE_CASE__      )
					self.assertTrue(isinstance(batch["""input_ids"""]						,	np.ndarray      )      )
					self.assertEqual(batch["""input_ids"""].tolist()						,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )
					_snake_case					:							Optional[int]					     =   tokenizer.pad(SCREAMING_SNAKE_CASE__						,	padding=SCREAMING_SNAKE_CASE__						,	return_tensors="""np"""      )
					self.assertTrue(isinstance(batch["""input_ids"""]						,	np.ndarray      )      )
					self.assertEqual(batch["""input_ids"""].tolist()						,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )







				@require_torch
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					import torch

					_snake_case					:							str					     =   [{"""input_ids""": torch.tensor([0, 1, 2]      )}, {"""input_ids""": torch.tensor([0, 1, 2, 3]      )}]
					_snake_case					:							Tuple					     =   BertTokenizer.from_pretrained("""bert-base-cased"""      )

					_snake_case					:							Dict					     =   tokenizer.pad(SCREAMING_SNAKE_CASE__						,	padding=SCREAMING_SNAKE_CASE__      )
					self.assertTrue(isinstance(batch["""input_ids"""]						,	torch.Tensor      )      )
					self.assertEqual(batch["""input_ids"""].tolist()						,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )
					_snake_case					:							Tuple					     =   tokenizer.pad(SCREAMING_SNAKE_CASE__						,	padding=SCREAMING_SNAKE_CASE__						,	return_tensors="""pt"""      )
					self.assertTrue(isinstance(batch["""input_ids"""]						,	torch.Tensor      )      )
					self.assertEqual(batch["""input_ids"""].tolist()						,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )







				@require_tf
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					import tensorflow as tf

					_snake_case					:							Optional[Any]					     =   [{"""input_ids""": tf.constant([0, 1, 2]      )}, {"""input_ids""": tf.constant([0, 1, 2, 3]      )}]
					_snake_case					:							Optional[Any]					     =   BertTokenizer.from_pretrained("""bert-base-cased"""      )

					_snake_case					:							Optional[Any]					     =   tokenizer.pad(SCREAMING_SNAKE_CASE__						,	padding=SCREAMING_SNAKE_CASE__      )
					self.assertTrue(isinstance(batch["""input_ids"""]						,	tf.Tensor      )      )
					self.assertEqual(batch["""input_ids"""].numpy().tolist()						,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )
					_snake_case					:							List[Any]					     =   tokenizer.pad(SCREAMING_SNAKE_CASE__						,	padding=SCREAMING_SNAKE_CASE__						,	return_tensors="""tf"""      )
					self.assertTrue(isinstance(batch["""input_ids"""]						,	tf.Tensor      )      )
					self.assertEqual(batch["""input_ids"""].numpy().tolist()						,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )







				@require_tokenizers
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							Optional[int]					     =   Tokenizer(WordPiece(unk_token="""[UNK]"""      )      )
					PreTrainedTokenizerFast(tokenizer_object=SCREAMING_SNAKE_CASE__      )







				@require_tokenizers
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							Optional[Any]					     =   Tokenizer(WordPiece(unk_token="""[UNK]"""      )      )
					with tempfile.TemporaryDirectory() as tmpdirname:
						bert_tokenizer.save(os.path.join(SCREAMING_SNAKE_CASE__						,	"""tokenizer.json"""      )      )
						PreTrainedTokenizerFast(tokenizer_file=os.path.join(SCREAMING_SNAKE_CASE__						,	"""tokenizer.json"""      )      )





