
import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
							from tokenizers import Tokenizer
							from tokenizers.models import WordPiece
class       A__  (		unittest.TestCase							):





							def      a__					(					self    :  Optional[Any]	,					_UpperCAmelCase    :  List[Any]							)		->						Any:



											"""simple docstring"""


											__lowercase								=   list(tokenizer_class.max_model_input_sizes.keys()							)
											for model_name in sa_models[:1]:
															__lowercase								=   tokenizer_class.from_pretrained(_UpperCAmelCase							)
															self.assertIsNotNone(_UpperCAmelCase							)
															self.assertIsInstance(_UpperCAmelCase	,					_UpperCAmelCase							)
															self.assertIsInstance(_UpperCAmelCase	,					_UpperCAmelCase							)

															for special_tok in tokenizer.all_special_tokens:
																			self.assertIsInstance(_UpperCAmelCase	,					_UpperCAmelCase							)
																			__lowercase								=   tokenizer.convert_tokens_to_ids(_UpperCAmelCase							)
																			self.assertIsInstance(_UpperCAmelCase	,					_UpperCAmelCase							)





							def      a__					(					self    :  List[str]	,					_UpperCAmelCase    :  BatchEncoding	,					_UpperCAmelCase    :  Optional[Callable] = None							)		->						Any:



											"""simple docstring"""


											__lowercase								=   pickle.dumps(_UpperCAmelCase							)
											self.assertIsNotNone(_UpperCAmelCase							)

											__lowercase								=   pickle.loads(_UpperCAmelCase							)

											# Ensure is_fast is correctly restored
											self.assertEqual(be_restored.is_fast	,					be_original.is_fast							)

											# Ensure encodings are potentially correctly restored
											if be_original.is_fast:
															self.assertIsNotNone(be_restored.encodings							)
											else:
															self.assertIsNone(be_restored.encodings							)

											# Ensure the keys are the same
											for original_v, restored_v in zip(be_original.values()	,					be_restored.values()							):
															if equal_op:
																			self.assertTrue(equal_op(_UpperCAmelCase	,					_UpperCAmelCase							)							)
															else:
																			self.assertEqual(_UpperCAmelCase	,					_UpperCAmelCase							)





							@slow
							def      a__					(					self    :  Tuple							)		->						Any:



											"""simple docstring"""


											self.check_tokenizer_from_pretrained(_UpperCAmelCase							)





							def      a__					(					self    :  Tuple							)		->						Tuple:



											"""simple docstring"""


											self.assertEqual(TensorType('tf'							)	,					TensorType.TENSORFLOW							)
											self.assertEqual(TensorType('pt'							)	,					TensorType.PYTORCH							)
											self.assertEqual(TensorType('np'							)	,					TensorType.NUMPY							)





							@require_tokenizers
							def      a__					(					self    :  Any							)		->						List[str]:



											"""simple docstring"""


											import numpy as np

											__lowercase								=   BertTokenizer.from_pretrained('bert-base-cased'							)
											__lowercase								=   BertTokenizerFast.from_pretrained('bert-base-cased'							)

											# Python no tensor
											with self.subTest('BatchEncoding (Python, return_tensors=None)'							):
															self.assert_dump_and_restore(tokenizer_p('Small example to encode'							)							)

											with self.subTest('BatchEncoding (Python, return_tensors=NUMPY)'							):
															self.assert_dump_and_restore(
															    tokenizer_p('Small example to encode'	,					return_tensors=TensorType.NUMPY							)	,					np.array_equal							)

											with self.subTest('BatchEncoding (Rust, return_tensors=None)'							):
															self.assert_dump_and_restore(tokenizer_r('Small example to encode'							)							)

											with self.subTest('BatchEncoding (Rust, return_tensors=NUMPY)'							):
															self.assert_dump_and_restore(
															    tokenizer_r('Small example to encode'	,					return_tensors=TensorType.NUMPY							)	,					np.array_equal							)





							@require_tf
							@require_tokenizers
							def      a__					(					self    :  Dict							)		->						Any:



											"""simple docstring"""


											import tensorflow as tf

											def tf_array_equals(_UpperCAmelCase    :  Optional[Any]	,					_UpperCAmelCase    :  int							):
															return tf.reduce_all(tf.equal(_UpperCAmelCase	,					_UpperCAmelCase							)							)

											__lowercase								=   BertTokenizer.from_pretrained('bert-base-cased'							)
											__lowercase								=   BertTokenizerFast.from_pretrained('bert-base-cased'							)

											with self.subTest('BatchEncoding (Python, return_tensors=TENSORFLOW)'							):
															self.assert_dump_and_restore(
															    tokenizer_p('Small example to encode'	,					return_tensors=TensorType.TENSORFLOW							)	,					_UpperCAmelCase							)

											with self.subTest('BatchEncoding (Rust, return_tensors=TENSORFLOW)'							):
															self.assert_dump_and_restore(
															    tokenizer_r('Small example to encode'	,					return_tensors=TensorType.TENSORFLOW							)	,					_UpperCAmelCase							)





							@require_torch
							@require_tokenizers
							def      a__					(					self    :  Union[str, Any]							)		->						List[str]:



											"""simple docstring"""


											import torch

											__lowercase								=   BertTokenizer.from_pretrained('bert-base-cased'							)
											__lowercase								=   BertTokenizerFast.from_pretrained('bert-base-cased'							)

											with self.subTest('BatchEncoding (Python, return_tensors=PYTORCH)'							):
															self.assert_dump_and_restore(
															    tokenizer_p('Small example to encode'	,					return_tensors=TensorType.PYTORCH							)	,					torch.equal							)

											with self.subTest('BatchEncoding (Rust, return_tensors=PYTORCH)'							):
															self.assert_dump_and_restore(
															    tokenizer_r('Small example to encode'	,					return_tensors=TensorType.PYTORCH							)	,					torch.equal							)





							@require_tokenizers
							def      a__					(					self    :  int							)		->						Any:



											"""simple docstring"""


											__lowercase								=   BertTokenizer.from_pretrained('bert-base-cased'							)
											__lowercase								=   BertTokenizerFast.from_pretrained('bert-base-cased'							)

											with self.subTest('Python Tokenizer'							):
															self.assertFalse(tokenizer_p('Small example to_encode'							).is_fast							)

											with self.subTest('Rust Tokenizer'							):
															self.assertTrue(tokenizer_r('Small example to_encode'							).is_fast							)





							@require_tokenizers
							def      a__					(					self    :  Any							)		->						Optional[int]:



											"""simple docstring"""


											__lowercase								=   BertTokenizerFast.from_pretrained('bert-base-cased'							)
											__lowercase								=   tokenizer_r(['Test', '\xad', 'test']	,					is_split_into_words=_UpperCAmelCase							)

											self.assertEqual(encoded.word_to_tokens(0							)	,					TokenSpan(start=1	,					end=2							)							)
											self.assertEqual(encoded.word_to_tokens(1							)	,					_UpperCAmelCase							)
											self.assertEqual(encoded.word_to_tokens(2							)	,					TokenSpan(start=2	,					end=3							)							)





							def      a__					(					self    :  List[Any]							)		->						int:



											"""simple docstring"""


											__lowercase								=   BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}							)
											__lowercase								=   batch.convert_to_tensors(tensor_type='np'							)
											self.assertEqual(tensor_batch['inputs'].shape	,					(2, 3)							)
											self.assertEqual(tensor_batch['labels'].shape	,					(2,)							)
											# test converting the converted
											with CaptureStderr() as cs:
															__lowercase								=   batch.convert_to_tensors(tensor_type='np'							)
											self.assertFalse(len(cs.err							)	,					msg=f"""should have no warning, but got {cs.err}"""							)

											__lowercase								=   BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}							)
											__lowercase								=   batch.convert_to_tensors(tensor_type='np'	,					prepend_batch_axis=_UpperCAmelCase							)
											self.assertEqual(tensor_batch['inputs'].shape	,					(1, 3)							)
											self.assertEqual(tensor_batch['labels'].shape	,					(1,)							)





							@require_torch
							def      a__					(					self    :  str							)		->						Union[str, Any]:



											"""simple docstring"""


											__lowercase								=   BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}							)
											__lowercase								=   batch.convert_to_tensors(tensor_type='pt'							)
											self.assertEqual(tensor_batch['inputs'].shape	,					(2, 3)							)
											self.assertEqual(tensor_batch['labels'].shape	,					(2,)							)
											# test converting the converted
											with CaptureStderr() as cs:
															__lowercase								=   batch.convert_to_tensors(tensor_type='pt'							)
											self.assertFalse(len(cs.err							)	,					msg=f"""should have no warning, but got {cs.err}"""							)

											__lowercase								=   BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}							)
											__lowercase								=   batch.convert_to_tensors(tensor_type='pt'	,					prepend_batch_axis=_UpperCAmelCase							)
											self.assertEqual(tensor_batch['inputs'].shape	,					(1, 3)							)
											self.assertEqual(tensor_batch['labels'].shape	,					(1,)							)





							@require_tf
							def      a__					(					self    :  Tuple							)		->						Optional[int]:



											"""simple docstring"""


											__lowercase								=   BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}							)
											__lowercase								=   batch.convert_to_tensors(tensor_type='tf'							)
											self.assertEqual(tensor_batch['inputs'].shape	,					(2, 3)							)
											self.assertEqual(tensor_batch['labels'].shape	,					(2,)							)
											# test converting the converted
											with CaptureStderr() as cs:
															__lowercase								=   batch.convert_to_tensors(tensor_type='tf'							)
											self.assertFalse(len(cs.err							)	,					msg=f"""should have no warning, but got {cs.err}"""							)

											__lowercase								=   BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}							)
											__lowercase								=   batch.convert_to_tensors(tensor_type='tf'	,					prepend_batch_axis=_UpperCAmelCase							)
											self.assertEqual(tensor_batch['inputs'].shape	,					(1, 3)							)
											self.assertEqual(tensor_batch['labels'].shape	,					(1,)							)





							@require_flax
							def      a__					(					self    :  Any							)		->						Any:



											"""simple docstring"""


											__lowercase								=   BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}							)
											__lowercase								=   batch.convert_to_tensors(tensor_type='jax'							)
											self.assertEqual(tensor_batch['inputs'].shape	,					(2, 3)							)
											self.assertEqual(tensor_batch['labels'].shape	,					(2,)							)
											# test converting the converted
											with CaptureStderr() as cs:
															__lowercase								=   batch.convert_to_tensors(tensor_type='jax'							)
											self.assertFalse(len(cs.err							)	,					msg=f"""should have no warning, but got {cs.err}"""							)

											__lowercase								=   BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}							)
											__lowercase								=   batch.convert_to_tensors(tensor_type='jax'	,					prepend_batch_axis=_UpperCAmelCase							)
											self.assertEqual(tensor_batch['inputs'].shape	,					(1, 3)							)
											self.assertEqual(tensor_batch['labels'].shape	,					(1,)							)





							def      a__					(					self    :  Union[str, Any]							)		->						Any:



											"""simple docstring"""


											__lowercase								=   [{'input_ids': np.array([0, 1, 2]							)}, {'input_ids': np.array([0, 1, 2, 3]							)}]
											__lowercase								=   BertTokenizer.from_pretrained('bert-base-cased'							)

											__lowercase								=   tokenizer.pad(_UpperCAmelCase	,					padding=_UpperCAmelCase							)
											self.assertTrue(isinstance(batch['input_ids']	,					np.ndarray							)							)
											self.assertEqual(batch['input_ids'].tolist()	,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
											__lowercase								=   tokenizer.pad(_UpperCAmelCase	,					padding=_UpperCAmelCase	,					return_tensors='np'							)
											self.assertTrue(isinstance(batch['input_ids']	,					np.ndarray							)							)
											self.assertEqual(batch['input_ids'].tolist()	,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)





							@require_torch
							def      a__					(					self    :  List[str]							)		->						Any:



											"""simple docstring"""


											import torch

											__lowercase								=   [{'input_ids': torch.tensor([0, 1, 2]							)}, {'input_ids': torch.tensor([0, 1, 2, 3]							)}]
											__lowercase								=   BertTokenizer.from_pretrained('bert-base-cased'							)

											__lowercase								=   tokenizer.pad(_UpperCAmelCase	,					padding=_UpperCAmelCase							)
											self.assertTrue(isinstance(batch['input_ids']	,					torch.Tensor							)							)
											self.assertEqual(batch['input_ids'].tolist()	,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
											__lowercase								=   tokenizer.pad(_UpperCAmelCase	,					padding=_UpperCAmelCase	,					return_tensors='pt'							)
											self.assertTrue(isinstance(batch['input_ids']	,					torch.Tensor							)							)
											self.assertEqual(batch['input_ids'].tolist()	,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)





							@require_tf
							def      a__					(					self    :  Optional[Any]							)		->						List[str]:



											"""simple docstring"""


											import tensorflow as tf

											__lowercase								=   [{'input_ids': tf.constant([0, 1, 2]							)}, {'input_ids': tf.constant([0, 1, 2, 3]							)}]
											__lowercase								=   BertTokenizer.from_pretrained('bert-base-cased'							)

											__lowercase								=   tokenizer.pad(_UpperCAmelCase	,					padding=_UpperCAmelCase							)
											self.assertTrue(isinstance(batch['input_ids']	,					tf.Tensor							)							)
											self.assertEqual(batch['input_ids'].numpy().tolist()	,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
											__lowercase								=   tokenizer.pad(_UpperCAmelCase	,					padding=_UpperCAmelCase	,					return_tensors='tf'							)
											self.assertTrue(isinstance(batch['input_ids']	,					tf.Tensor							)							)
											self.assertEqual(batch['input_ids'].numpy().tolist()	,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)





							@require_tokenizers
							def      a__					(					self    :  Dict							)		->						Tuple:



											"""simple docstring"""


											__lowercase								=   Tokenizer(WordPiece(unk_token='[UNK]'							)							)
											PreTrainedTokenizerFast(tokenizer_object=_UpperCAmelCase							)





							@require_tokenizers
							def      a__					(					self    :  str							)		->						Dict:



											"""simple docstring"""


											__lowercase								=   Tokenizer(WordPiece(unk_token='[UNK]'							)							)
											with tempfile.TemporaryDirectory() as tmpdirname:
															bert_tokenizer.save(os.path.join(_UpperCAmelCase	,					'tokenizer.json'							)							)
															PreTrainedTokenizerFast(tokenizer_file=os.path.join(_UpperCAmelCase	,					'tokenizer.json'							)							)







from ..utils import DummyObject, requires_backends
class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Tuple            =				["tokenizers"]





							def __init__(					self    :  str	,					*_UpperCAmelCase    :  List[str]	,					**_UpperCAmelCase    :  List[str]							)		->						int:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Dict            =				["tokenizers"]





							def __init__(					self    :  str	,					*_UpperCAmelCase    :  Union[str, Any]	,					**_UpperCAmelCase    :  Tuple							)		->						Optional[Any]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Union[str, Any]            =				["tokenizers"]





							def __init__(					self    :  Union[str, Any]	,					*_UpperCAmelCase    :  Union[str, Any]	,					**_UpperCAmelCase    :  int							)		->						Dict:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Optional[int]            =				["tokenizers"]





							def __init__(					self    :  int	,					*_UpperCAmelCase    :  Dict	,					**_UpperCAmelCase    :  Optional[Any]							)		->						List[str]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		str            =				["tokenizers"]





							def __init__(					self    :  Optional[Any]	,					*_UpperCAmelCase    :  Optional[int]	,					**_UpperCAmelCase    :  Optional[int]							)		->						Optional[int]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Optional[int]            =				["tokenizers"]





							def __init__(					self    :  Any	,					*_UpperCAmelCase    :  Union[str, Any]	,					**_UpperCAmelCase    :  List[str]							)		->						Tuple:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Optional[Any]            =				["tokenizers"]





							def __init__(					self    :  str	,					*_UpperCAmelCase    :  Optional[int]	,					**_UpperCAmelCase    :  Union[str, Any]							)		->						Optional[Any]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		List[str]            =				["tokenizers"]





							def __init__(					self    :  Optional[int]	,					*_UpperCAmelCase    :  Union[str, Any]	,					**_UpperCAmelCase    :  List[str]							)		->						Optional[Any]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Any            =				["tokenizers"]





							def __init__(					self    :  Optional[Any]	,					*_UpperCAmelCase    :  Tuple	,					**_UpperCAmelCase    :  Any							)		->						Optional[Any]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Dict            =				["tokenizers"]





							def __init__(					self    :  List[str]	,					*_UpperCAmelCase    :  Tuple	,					**_UpperCAmelCase    :  List[str]							)		->						List[Any]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		List[Any]            =				["tokenizers"]





							def __init__(					self    :  Tuple	,					*_UpperCAmelCase    :  Union[str, Any]	,					**_UpperCAmelCase    :  Any							)		->						Any:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Tuple            =				["tokenizers"]





							def __init__(					self    :  int	,					*_UpperCAmelCase    :  int	,					**_UpperCAmelCase    :  Optional[Any]							)		->						List[str]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Optional[int]            =				["tokenizers"]





							def __init__(					self    :  Tuple	,					*_UpperCAmelCase    :  List[Any]	,					**_UpperCAmelCase    :  int							)		->						Tuple:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Tuple            =				["tokenizers"]





							def __init__(					self    :  List[str]	,					*_UpperCAmelCase    :  Any	,					**_UpperCAmelCase    :  str							)		->						int:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		str            =				["tokenizers"]





							def __init__(					self    :  Any	,					*_UpperCAmelCase    :  str	,					**_UpperCAmelCase    :  List[Any]							)		->						List[str]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Dict            =				["tokenizers"]





							def __init__(					self    :  int	,					*_UpperCAmelCase    :  Tuple	,					**_UpperCAmelCase    :  Optional[Any]							)		->						Dict:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Tuple            =				["tokenizers"]





							def __init__(					self    :  Tuple	,					*_UpperCAmelCase    :  str	,					**_UpperCAmelCase    :  List[Any]							)		->						Union[str, Any]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Optional[Any]            =				["tokenizers"]





							def __init__(					self    :  Dict	,					*_UpperCAmelCase    :  Optional[Any]	,					**_UpperCAmelCase    :  List[Any]							)		->						Optional[Any]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Dict            =				["tokenizers"]





							def __init__(					self    :  Optional[Any]	,					*_UpperCAmelCase    :  Optional[int]	,					**_UpperCAmelCase    :  List[str]							)		->						Optional[int]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Tuple            =				["tokenizers"]





							def __init__(					self    :  Union[str, Any]	,					*_UpperCAmelCase    :  Optional[int]	,					**_UpperCAmelCase    :  Any							)		->						Tuple:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		List[str]            =				["tokenizers"]





							def __init__(					self    :  Union[str, Any]	,					*_UpperCAmelCase    :  List[Any]	,					**_UpperCAmelCase    :  Any							)		->						Tuple:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		int            =				["tokenizers"]





							def __init__(					self    :  Optional[Any]	,					*_UpperCAmelCase    :  Tuple	,					**_UpperCAmelCase    :  Tuple							)		->						List[str]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		int            =				["tokenizers"]





							def __init__(					self    :  Tuple	,					*_UpperCAmelCase    :  Tuple	,					**_UpperCAmelCase    :  Tuple							)		->						Tuple:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		List[str]            =				["tokenizers"]





							def __init__(					self    :  Tuple	,					*_UpperCAmelCase    :  List[Any]	,					**_UpperCAmelCase    :  Tuple							)		->						List[Any]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		List[str]            =				["tokenizers"]





							def __init__(					self    :  int	,					*_UpperCAmelCase    :  Optional[Any]	,					**_UpperCAmelCase    :  List[Any]							)		->						List[Any]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Optional[int]            =				["tokenizers"]





							def __init__(					self    :  Any	,					*_UpperCAmelCase    :  Optional[Any]	,					**_UpperCAmelCase    :  str							)		->						Tuple:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		str            =				["tokenizers"]





							def __init__(					self    :  Optional[int]	,					*_UpperCAmelCase    :  Any	,					**_UpperCAmelCase    :  Union[str, Any]							)		->						int:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Dict            =				["tokenizers"]





							def __init__(					self    :  Union[str, Any]	,					*_UpperCAmelCase    :  Dict	,					**_UpperCAmelCase    :  Any							)		->						Any:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Union[str, Any]            =				["tokenizers"]





							def __init__(					self    :  Any	,					*_UpperCAmelCase    :  Tuple	,					**_UpperCAmelCase    :  Optional[int]							)		->						int:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Union[str, Any]            =				["tokenizers"]





							def __init__(					self    :  Tuple	,					*_UpperCAmelCase    :  Optional[int]	,					**_UpperCAmelCase    :  Tuple							)		->						List[str]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Optional[int]            =				["tokenizers"]





							def __init__(					self    :  Dict	,					*_UpperCAmelCase    :  Optional[Any]	,					**_UpperCAmelCase    :  Optional[int]							)		->						Dict:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Tuple            =				["tokenizers"]





							def __init__(					self    :  Optional[int]	,					*_UpperCAmelCase    :  Dict	,					**_UpperCAmelCase    :  str							)		->						Optional[int]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Tuple            =				["tokenizers"]





							def __init__(					self    :  Optional[int]	,					*_UpperCAmelCase    :  Optional[Any]	,					**_UpperCAmelCase    :  List[Any]							)		->						Optional[Any]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		int            =				["tokenizers"]





							def __init__(					self    :  Optional[Any]	,					*_UpperCAmelCase    :  Tuple	,					**_UpperCAmelCase    :  Any							)		->						str:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Tuple            =				["tokenizers"]





							def __init__(					self    :  Dict	,					*_UpperCAmelCase    :  List[Any]	,					**_UpperCAmelCase    :  Tuple							)		->						Optional[Any]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Any            =				["tokenizers"]





							def __init__(					self    :  int	,					*_UpperCAmelCase    :  str	,					**_UpperCAmelCase    :  Any							)		->						Optional[int]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		List[Any]            =				["tokenizers"]





							def __init__(					self    :  Any	,					*_UpperCAmelCase    :  Any	,					**_UpperCAmelCase    :  Tuple							)		->						List[str]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Any            =				["tokenizers"]





							def __init__(					self    :  Union[str, Any]	,					*_UpperCAmelCase    :  Optional[Any]	,					**_UpperCAmelCase    :  Dict							)		->						str:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Union[str, Any]            =				["tokenizers"]





							def __init__(					self    :  List[str]	,					*_UpperCAmelCase    :  List[str]	,					**_UpperCAmelCase    :  List[str]							)		->						Union[str, Any]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Optional[int]            =				["tokenizers"]





							def __init__(					self    :  Dict	,					*_UpperCAmelCase    :  List[Any]	,					**_UpperCAmelCase    :  List[Any]							)		->						Union[str, Any]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Optional[Any]            =				["tokenizers"]





							def __init__(					self    :  Optional[int]	,					*_UpperCAmelCase    :  Optional[int]	,					**_UpperCAmelCase    :  str							)		->						Tuple:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Any            =				["tokenizers"]





							def __init__(					self    :  Optional[int]	,					*_UpperCAmelCase    :  str	,					**_UpperCAmelCase    :  Optional[int]							)		->						Optional[int]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		List[str]            =				["tokenizers"]





							def __init__(					self    :  Tuple	,					*_UpperCAmelCase    :  Tuple	,					**_UpperCAmelCase    :  Optional[Any]							)		->						Tuple:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Optional[Any]            =				["tokenizers"]





							def __init__(					self    :  int	,					*_UpperCAmelCase    :  Tuple	,					**_UpperCAmelCase    :  Dict							)		->						Union[str, Any]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Union[str, Any]            =				["tokenizers"]





							def __init__(					self    :  List[str]	,					*_UpperCAmelCase    :  Optional[Any]	,					**_UpperCAmelCase    :  str							)		->						Union[str, Any]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Optional[Any]            =				["tokenizers"]





							def __init__(					self    :  Dict	,					*_UpperCAmelCase    :  Tuple	,					**_UpperCAmelCase    :  List[str]							)		->						Any:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		int            =				["tokenizers"]





							def __init__(					self    :  Optional[Any]	,					*_UpperCAmelCase    :  int	,					**_UpperCAmelCase    :  Optional[int]							)		->						Dict:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Tuple            =				["tokenizers"]





							def __init__(					self    :  Optional[int]	,					*_UpperCAmelCase    :  List[str]	,					**_UpperCAmelCase    :  List[Any]							)		->						List[str]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Any            =				["tokenizers"]





							def __init__(					self    :  Optional[int]	,					*_UpperCAmelCase    :  Any	,					**_UpperCAmelCase    :  Tuple							)		->						Tuple:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Optional[int]            =				["tokenizers"]





							def __init__(					self    :  Any	,					*_UpperCAmelCase    :  Optional[int]	,					**_UpperCAmelCase    :  Any							)		->						int:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Tuple            =				["tokenizers"]





							def __init__(					self    :  Tuple	,					*_UpperCAmelCase    :  Any	,					**_UpperCAmelCase    :  Any							)		->						List[str]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		int            =				["tokenizers"]





							def __init__(					self    :  int	,					*_UpperCAmelCase    :  Any	,					**_UpperCAmelCase    :  Any							)		->						str:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		List[Any]            =				["tokenizers"]





							def __init__(					self    :  int	,					*_UpperCAmelCase    :  List[Any]	,					**_UpperCAmelCase    :  Any							)		->						Any:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Optional[int]            =				["tokenizers"]





							def __init__(					self    :  List[Any]	,					*_UpperCAmelCase    :  Any	,					**_UpperCAmelCase    :  Dict							)		->						Optional[Any]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Optional[Any]            =				["tokenizers"]





							def __init__(					self    :  Optional[int]	,					*_UpperCAmelCase    :  List[Any]	,					**_UpperCAmelCase    :  Optional[int]							)		->						List[str]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Any            =				["tokenizers"]





							def __init__(					self    :  Tuple	,					*_UpperCAmelCase    :  List[str]	,					**_UpperCAmelCase    :  Optional[int]							)		->						List[str]:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Any            =				["tokenizers"]





							def __init__(					self    :  List[str]	,					*_UpperCAmelCase    :  Optional[Any]	,					**_UpperCAmelCase    :  str							)		->						int:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)






class       A__  (		metaclass=lowerCAmelCase__							):
							lowerCAmelCase__						:		Dict            =				["tokenizers"]





							def __init__(					self    :  Optional[Any]	,					*_UpperCAmelCase    :  Tuple	,					**_UpperCAmelCase    :  str							)		->						Any:



											"""simple docstring"""


											requires_backends(self	,					['tokenizers']							)





