




import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
				from tokenizers import Tokenizer
				from tokenizers.models import WordPiece






class        __lowerCAmelCase					(			unittest.TestCase					):







			def UpperCamelCase     (			self       :     Dict       ,       snake_case__       :     Tuple ):




									"""simple docstring"""


									_UpperCAmelCase						=	list(tokenizer_class.max_model_input_sizes.keys() )
									for model_name in sa_models[:1]:
															_UpperCAmelCase						=	tokenizer_class.from_pretrained(snake_case__ )
															self.assertIsNotNone(snake_case__ )
															self.assertIsInstance(snake_case__       ,       snake_case__ )
															self.assertIsInstance(snake_case__       ,       snake_case__ )

															for special_tok in tokenizer.all_special_tokens:
																					self.assertIsInstance(snake_case__       ,       snake_case__ )
																					_UpperCAmelCase						=	tokenizer.convert_tokens_to_ids(snake_case__ )
																					self.assertIsInstance(snake_case__       ,       snake_case__ )







			def UpperCamelCase     (			self       :     Tuple       ,       snake_case__       :     BatchEncoding       ,       snake_case__       :     Optional[Callable] = None ):




									"""simple docstring"""


									_UpperCAmelCase						=	pickle.dumps(snake_case__ )
									self.assertIsNotNone(snake_case__ )

									_UpperCAmelCase						=	pickle.loads(snake_case__ )

									# Ensure is_fast is correctly restored
									self.assertEqual(be_restored.is_fast       ,       be_original.is_fast )

									# Ensure encodings are potentially correctly restored
									if be_original.is_fast:
															self.assertIsNotNone(be_restored.encodings )
									else:
															self.assertIsNone(be_restored.encodings )

									# Ensure the keys are the same
									for original_v, restored_v in zip(be_original.values()       ,       be_restored.values() ):
															if equal_op:
																					self.assertTrue(equal_op(snake_case__       ,       snake_case__ ) )
															else:
																					self.assertEqual(snake_case__       ,       snake_case__ )







			@slow
			def UpperCamelCase     (			self       :     Optional[Any] ):




									"""simple docstring"""


									self.check_tokenizer_from_pretrained(snake_case__ )







			def UpperCamelCase     (			self       :     int ):




									"""simple docstring"""


									self.assertEqual(TensorType("tf" )       ,       TensorType.TENSORFLOW )
									self.assertEqual(TensorType("pt" )       ,       TensorType.PYTORCH )
									self.assertEqual(TensorType("np" )       ,       TensorType.NUMPY )







			@require_tokenizers
			def UpperCamelCase     (			self       :     Optional[Any] ):




									"""simple docstring"""


									import numpy as np

									_UpperCAmelCase						=	BertTokenizer.from_pretrained("bert-base-cased" )
									_UpperCAmelCase						=	BertTokenizerFast.from_pretrained("bert-base-cased" )

									# Python no tensor
									with self.subTest("BatchEncoding (Python, return_tensors=None)" ):
															self.assert_dump_and_restore(tokenizer_p("Small example to encode" ) )

									with self.subTest("BatchEncoding (Python, return_tensors=NUMPY)" ):
															self.assert_dump_and_restore(
															    tokenizer_p("Small example to encode"       ,       return_tensors=TensorType.NUMPY )       ,       np.array_equal )

									with self.subTest("BatchEncoding (Rust, return_tensors=None)" ):
															self.assert_dump_and_restore(tokenizer_r("Small example to encode" ) )

									with self.subTest("BatchEncoding (Rust, return_tensors=NUMPY)" ):
															self.assert_dump_and_restore(
															    tokenizer_r("Small example to encode"       ,       return_tensors=TensorType.NUMPY )       ,       np.array_equal )







			@require_tf
			@require_tokenizers
			def UpperCamelCase     (			self       :     Dict ):




									"""simple docstring"""


									import tensorflow as tf

									def tf_array_equals(snake_case__       :     List[Any]       ,       snake_case__       :     str ):
															return tf.reduce_all(tf.equal(snake_case__       ,       snake_case__ ) )

									_UpperCAmelCase						=	BertTokenizer.from_pretrained("bert-base-cased" )
									_UpperCAmelCase						=	BertTokenizerFast.from_pretrained("bert-base-cased" )

									with self.subTest("BatchEncoding (Python, return_tensors=TENSORFLOW)" ):
															self.assert_dump_and_restore(
															    tokenizer_p("Small example to encode"       ,       return_tensors=TensorType.TENSORFLOW )       ,       snake_case__ )

									with self.subTest("BatchEncoding (Rust, return_tensors=TENSORFLOW)" ):
															self.assert_dump_and_restore(
															    tokenizer_r("Small example to encode"       ,       return_tensors=TensorType.TENSORFLOW )       ,       snake_case__ )







			@require_torch
			@require_tokenizers
			def UpperCamelCase     (			self       :     Optional[int] ):




									"""simple docstring"""


									import torch

									_UpperCAmelCase						=	BertTokenizer.from_pretrained("bert-base-cased" )
									_UpperCAmelCase						=	BertTokenizerFast.from_pretrained("bert-base-cased" )

									with self.subTest("BatchEncoding (Python, return_tensors=PYTORCH)" ):
															self.assert_dump_and_restore(
															    tokenizer_p("Small example to encode"       ,       return_tensors=TensorType.PYTORCH )       ,       torch.equal )

									with self.subTest("BatchEncoding (Rust, return_tensors=PYTORCH)" ):
															self.assert_dump_and_restore(
															    tokenizer_r("Small example to encode"       ,       return_tensors=TensorType.PYTORCH )       ,       torch.equal )







			@require_tokenizers
			def UpperCamelCase     (			self       :     Optional[Any] ):




									"""simple docstring"""


									_UpperCAmelCase						=	BertTokenizer.from_pretrained("bert-base-cased" )
									_UpperCAmelCase						=	BertTokenizerFast.from_pretrained("bert-base-cased" )

									with self.subTest("Python Tokenizer" ):
															self.assertFalse(tokenizer_p("Small example to_encode" ).is_fast )

									with self.subTest("Rust Tokenizer" ):
															self.assertTrue(tokenizer_r("Small example to_encode" ).is_fast )







			@require_tokenizers
			def UpperCamelCase     (			self       :     Optional[int] ):




									"""simple docstring"""


									_UpperCAmelCase						=	BertTokenizerFast.from_pretrained("bert-base-cased" )
									_UpperCAmelCase						=	tokenizer_r(["Test", "\xad", "test"]       ,       is_split_into_words=snake_case__ )

									self.assertEqual(encoded.word_to_tokens(0 )       ,       TokenSpan(start=1       ,       end=2 ) )
									self.assertEqual(encoded.word_to_tokens(1 )       ,       snake_case__ )
									self.assertEqual(encoded.word_to_tokens(2 )       ,       TokenSpan(start=2       ,       end=3 ) )







			def UpperCamelCase     (			self       :     Optional[Any] ):




									"""simple docstring"""


									_UpperCAmelCase						=	BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]} )
									_UpperCAmelCase						=	batch.convert_to_tensors(tensor_type="np" )
									self.assertEqual(tensor_batch["inputs"].shape       ,       (2, 3) )
									self.assertEqual(tensor_batch["labels"].shape       ,       (2,) )
									# test converting the converted
									with CaptureStderr() as cs:
															_UpperCAmelCase						=	batch.convert_to_tensors(tensor_type="np" )
									self.assertFalse(len(cs.err )       ,       msg=F"""should have no warning, but got {cs.err}""" )

									_UpperCAmelCase						=	BatchEncoding({"inputs": [1, 2, 3], "labels": 0} )
									_UpperCAmelCase						=	batch.convert_to_tensors(tensor_type="np"       ,       prepend_batch_axis=snake_case__ )
									self.assertEqual(tensor_batch["inputs"].shape       ,       (1, 3) )
									self.assertEqual(tensor_batch["labels"].shape       ,       (1,) )







			@require_torch
			def UpperCamelCase     (			self       :     Tuple ):




									"""simple docstring"""


									_UpperCAmelCase						=	BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]} )
									_UpperCAmelCase						=	batch.convert_to_tensors(tensor_type="pt" )
									self.assertEqual(tensor_batch["inputs"].shape       ,       (2, 3) )
									self.assertEqual(tensor_batch["labels"].shape       ,       (2,) )
									# test converting the converted
									with CaptureStderr() as cs:
															_UpperCAmelCase						=	batch.convert_to_tensors(tensor_type="pt" )
									self.assertFalse(len(cs.err )       ,       msg=F"""should have no warning, but got {cs.err}""" )

									_UpperCAmelCase						=	BatchEncoding({"inputs": [1, 2, 3], "labels": 0} )
									_UpperCAmelCase						=	batch.convert_to_tensors(tensor_type="pt"       ,       prepend_batch_axis=snake_case__ )
									self.assertEqual(tensor_batch["inputs"].shape       ,       (1, 3) )
									self.assertEqual(tensor_batch["labels"].shape       ,       (1,) )







			@require_tf
			def UpperCamelCase     (			self       :     Tuple ):




									"""simple docstring"""


									_UpperCAmelCase						=	BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]} )
									_UpperCAmelCase						=	batch.convert_to_tensors(tensor_type="tf" )
									self.assertEqual(tensor_batch["inputs"].shape       ,       (2, 3) )
									self.assertEqual(tensor_batch["labels"].shape       ,       (2,) )
									# test converting the converted
									with CaptureStderr() as cs:
															_UpperCAmelCase						=	batch.convert_to_tensors(tensor_type="tf" )
									self.assertFalse(len(cs.err )       ,       msg=F"""should have no warning, but got {cs.err}""" )

									_UpperCAmelCase						=	BatchEncoding({"inputs": [1, 2, 3], "labels": 0} )
									_UpperCAmelCase						=	batch.convert_to_tensors(tensor_type="tf"       ,       prepend_batch_axis=snake_case__ )
									self.assertEqual(tensor_batch["inputs"].shape       ,       (1, 3) )
									self.assertEqual(tensor_batch["labels"].shape       ,       (1,) )







			@require_flax
			def UpperCamelCase     (			self       :     List[Any] ):




									"""simple docstring"""


									_UpperCAmelCase						=	BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]} )
									_UpperCAmelCase						=	batch.convert_to_tensors(tensor_type="jax" )
									self.assertEqual(tensor_batch["inputs"].shape       ,       (2, 3) )
									self.assertEqual(tensor_batch["labels"].shape       ,       (2,) )
									# test converting the converted
									with CaptureStderr() as cs:
															_UpperCAmelCase						=	batch.convert_to_tensors(tensor_type="jax" )
									self.assertFalse(len(cs.err )       ,       msg=F"""should have no warning, but got {cs.err}""" )

									_UpperCAmelCase						=	BatchEncoding({"inputs": [1, 2, 3], "labels": 0} )
									_UpperCAmelCase						=	batch.convert_to_tensors(tensor_type="jax"       ,       prepend_batch_axis=snake_case__ )
									self.assertEqual(tensor_batch["inputs"].shape       ,       (1, 3) )
									self.assertEqual(tensor_batch["labels"].shape       ,       (1,) )







			def UpperCamelCase     (			self       :     List[str] ):




									"""simple docstring"""


									_UpperCAmelCase						=	[{"input_ids": np.array([0, 1, 2] )}, {"input_ids": np.array([0, 1, 2, 3] )}]
									_UpperCAmelCase						=	BertTokenizer.from_pretrained("bert-base-cased" )

									_UpperCAmelCase						=	tokenizer.pad(snake_case__       ,       padding=snake_case__ )
									self.assertTrue(isinstance(batch["input_ids"]       ,       np.ndarray ) )
									self.assertEqual(batch["input_ids"].tolist()       ,       [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
									_UpperCAmelCase						=	tokenizer.pad(snake_case__       ,       padding=snake_case__       ,       return_tensors="np" )
									self.assertTrue(isinstance(batch["input_ids"]       ,       np.ndarray ) )
									self.assertEqual(batch["input_ids"].tolist()       ,       [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )







			@require_torch
			def UpperCamelCase     (			self       :     List[Any] ):




									"""simple docstring"""


									import torch

									_UpperCAmelCase						=	[{"input_ids": torch.tensor([0, 1, 2] )}, {"input_ids": torch.tensor([0, 1, 2, 3] )}]
									_UpperCAmelCase						=	BertTokenizer.from_pretrained("bert-base-cased" )

									_UpperCAmelCase						=	tokenizer.pad(snake_case__       ,       padding=snake_case__ )
									self.assertTrue(isinstance(batch["input_ids"]       ,       torch.Tensor ) )
									self.assertEqual(batch["input_ids"].tolist()       ,       [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
									_UpperCAmelCase						=	tokenizer.pad(snake_case__       ,       padding=snake_case__       ,       return_tensors="pt" )
									self.assertTrue(isinstance(batch["input_ids"]       ,       torch.Tensor ) )
									self.assertEqual(batch["input_ids"].tolist()       ,       [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )







			@require_tf
			def UpperCamelCase     (			self       :     int ):




									"""simple docstring"""


									import tensorflow as tf

									_UpperCAmelCase						=	[{"input_ids": tf.constant([0, 1, 2] )}, {"input_ids": tf.constant([0, 1, 2, 3] )}]
									_UpperCAmelCase						=	BertTokenizer.from_pretrained("bert-base-cased" )

									_UpperCAmelCase						=	tokenizer.pad(snake_case__       ,       padding=snake_case__ )
									self.assertTrue(isinstance(batch["input_ids"]       ,       tf.Tensor ) )
									self.assertEqual(batch["input_ids"].numpy().tolist()       ,       [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
									_UpperCAmelCase						=	tokenizer.pad(snake_case__       ,       padding=snake_case__       ,       return_tensors="tf" )
									self.assertTrue(isinstance(batch["input_ids"]       ,       tf.Tensor ) )
									self.assertEqual(batch["input_ids"].numpy().tolist()       ,       [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )







			@require_tokenizers
			def UpperCamelCase     (			self       :     Any ):




									"""simple docstring"""


									_UpperCAmelCase						=	Tokenizer(WordPiece(unk_token="[UNK]" ) )
									PreTrainedTokenizerFast(tokenizer_object=snake_case__ )







			@require_tokenizers
			def UpperCamelCase     (			self       :     int ):




									"""simple docstring"""


									_UpperCAmelCase						=	Tokenizer(WordPiece(unk_token="[UNK]" ) )
									with tempfile.TemporaryDirectory() as tmpdirname:
															bert_tokenizer.save(os.path.join(snake_case__       ,       "tokenizer.json" ) )
															PreTrainedTokenizerFast(tokenizer_file=os.path.join(snake_case__       ,       "tokenizer.json" ) )










import enum
import inspect
import warnings
from functools import wraps
from typing import Callable, Optional

from .logging import get_logger


lowercase_      :							Optional[int]    							=					set()
lowercase_      :							List[Any]    							=					get_logger(__name__)


def 				__SCREAMING_SNAKE_CASE    (     snake_case_ = None		):

						'''simple docstring'''


						def decorator(snake_case_		):
												global _emitted_deprecation_warnings

												if inspect.isclass(snake_case_		):
																		_UpperCAmelCase						=	deprecated_class_or_function.__init__
																		_UpperCAmelCase						=	deprecated_class_or_function.__name__
												else:
																		_UpperCAmelCase						=	deprecated_class_or_function
																		_UpperCAmelCase						=	deprecated_function.__name__
																		# Support deprecating __init__ class method: class name instead
																		_UpperCAmelCase						=	name if name != "__init__" else deprecated_function.__qualname__.split("."		)[-2]

												_UpperCAmelCase						=	(
												    f"""{name} is deprecated and will be removed in the next major version of datasets.""" + f""" {help_message}"""
												    if help_message
												    else ""
												)

												@wraps(snake_case_		)
												def wrapper(*snake_case_      ,       **snake_case_		):
																		_UpperCAmelCase						=	hash(snake_case_		)
																		if func_hash not in _emitted_deprecation_warnings:
																								warnings.warn(snake_case_      ,       category=snake_case_      ,       stacklevel=2		)
																								_emitted_deprecation_warnings.add(snake_case_		)
																		return deprecated_function(*snake_case_      ,       **snake_case_		)

												_UpperCAmelCase						=	"deprecated"

												if inspect.isclass(snake_case_		):
																		_UpperCAmelCase						=	wrapper
																		return deprecated_class_or_function
												else:
																		return wrapper

						return decorator






class        __lowerCAmelCase					(			enum.EnumMeta					):







			def __getattribute__(			cls       :     str       ,       snake_case__       :     Optional[Any] ):




									"""simple docstring"""


									_UpperCAmelCase						=	super().__getattribute__(snake_case__ )
									if isinstance(snake_case__       ,       enum.Enum ) and obj._on_access:
															obj._on_access()
									return obj







			def __getitem__(			cls       :     str       ,       snake_case__       :     Union[str, Any] ):




									"""simple docstring"""


									_UpperCAmelCase						=	super().__getitem__(snake_case__ )
									if member._on_access:
															member._on_access()
									return member







			def __call__(			cls       :     Dict       ,       snake_case__       :     Any       ,       snake_case__       :     List[str]=None       ,       *, snake_case__       :     Optional[int]=None       ,       snake_case__       :     List[Any]=None       ,       snake_case__       :     Tuple=None       ,       snake_case__       :     List[Any]=1 ):




									"""simple docstring"""


									_UpperCAmelCase						=	super().__call__(snake_case__       ,       snake_case__       ,       module=snake_case__       ,       qualname=snake_case__       ,       type=snake_case__       ,       start=snake_case__ )
									if isinstance(snake_case__       ,       enum.Enum ) and obj._on_access:
															obj._on_access()
									return obj






class        __lowerCAmelCase					(			enum.Enum   ,						metaclass=UpperCAmelCase__					):







			def __new__(			cls       :     Any       ,       snake_case__       :     Dict ):




									"""simple docstring"""


									_UpperCAmelCase						=	object.__new__(cls )
									_UpperCAmelCase						=	value
									_UpperCAmelCase						=	member.deprecate
									return member







			@property
			def UpperCamelCase     (			self       :     str ):




									"""simple docstring"""


									return ""







			def UpperCamelCase     (			self       :     Optional[int] ):




									"""simple docstring"""


									_UpperCAmelCase						=	F""" {self.help_message}""" if self.help_message else ""
									warnings.warn(
									    F"""'{self.__objclass__.__name__}' is deprecated and will be removed in the next major version of datasets."""
									    + help_message       ,       snake_case__       ,       stacklevel=3       ,       )




