




import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
							from tokenizers import Tokenizer
							from tokenizers.models import WordPiece


class 							__SCREAMING_SNAKE_CASE				(						unittest.TestCase    ):

					def       lowercase_				(							self					,			__lowercase   ):







						'''simple docstring'''




						lowerCamelCase_				:						Any				  =				list(tokenizer_class.max_model_input_sizes.keys()   )
						for model_name in sa_models[:1]:
							lowerCamelCase_				:						Union[str, Any]				  =				tokenizer_class.from_pretrained(__lowercase   )
							self.assertIsNotNone(__lowercase   )
							self.assertIsInstance(__lowercase					,			__lowercase   )
							self.assertIsInstance(__lowercase					,			__lowercase   )

							for special_tok in tokenizer.all_special_tokens:
								self.assertIsInstance(__lowercase					,			__lowercase   )
								lowerCamelCase_				:						List[str]				  =				tokenizer.convert_tokens_to_ids(__lowercase   )
								self.assertIsInstance(__lowercase					,			__lowercase   )

					def       lowercase_				(							self					,			__lowercase					,			__lowercase = None   ):







						'''simple docstring'''




						lowerCamelCase_				:						Union[str, Any]				  =				pickle.dumps(__lowercase   )
						self.assertIsNotNone(__lowercase   )

						lowerCamelCase_				:						int				  =				pickle.loads(__lowercase   )

						# Ensure is_fast is correctly restored
						self.assertEqual(be_restored.is_fast					,			be_original.is_fast   )

						# Ensure encodings are potentially correctly restored
						if be_original.is_fast:
							self.assertIsNotNone(be_restored.encodings   )
						else:
							self.assertIsNone(be_restored.encodings   )

						# Ensure the keys are the same
						for original_v, restored_v in zip(be_original.values()					,			be_restored.values()   ):
							if equal_op:
								self.assertTrue(equal_op(__lowercase					,			__lowercase   )   )
							else:
								self.assertEqual(__lowercase					,			__lowercase   )

					@slow
					def       lowercase_				(							self   ):







						'''simple docstring'''




						self.check_tokenizer_from_pretrained(__lowercase   )

					def       lowercase_				(							self   ):







						'''simple docstring'''




						self.assertEqual(TensorType('tf'   )					,			TensorType.TENSORFLOW   )
						self.assertEqual(TensorType('pt'   )					,			TensorType.PYTORCH   )
						self.assertEqual(TensorType('np'   )					,			TensorType.NUMPY   )

					@require_tokenizers
					def       lowercase_				(							self   ):







						'''simple docstring'''




						import numpy as np

						lowerCamelCase_				:						Tuple				  =				BertTokenizer.from_pretrained('bert-base-cased'   )
						lowerCamelCase_				:						int				  =				BertTokenizerFast.from_pretrained('bert-base-cased'   )

						# Python no tensor
						with self.subTest('BatchEncoding (Python, return_tensors=None)'   ):
							self.assert_dump_and_restore(tokenizer_p('Small example to encode'   )   )

						with self.subTest('BatchEncoding (Python, return_tensors=NUMPY)'   ):
							self.assert_dump_and_restore(
							    tokenizer_p('Small example to encode'					,			return_tensors=TensorType.NUMPY   )					,			np.array_equal   )

						with self.subTest('BatchEncoding (Rust, return_tensors=None)'   ):
							self.assert_dump_and_restore(tokenizer_r('Small example to encode'   )   )

						with self.subTest('BatchEncoding (Rust, return_tensors=NUMPY)'   ):
							self.assert_dump_and_restore(
							    tokenizer_r('Small example to encode'					,			return_tensors=TensorType.NUMPY   )					,			np.array_equal   )

					@require_tf
					@require_tokenizers
					def       lowercase_				(							self   ):







						'''simple docstring'''




						import tensorflow as tf

						def tf_array_equals(__lowercase					,			__lowercase   ):
							return tf.reduce_all(tf.equal(__lowercase					,			__lowercase   )   )

						lowerCamelCase_				:						List[Any]				  =				BertTokenizer.from_pretrained('bert-base-cased'   )
						lowerCamelCase_				:						Dict				  =				BertTokenizerFast.from_pretrained('bert-base-cased'   )

						with self.subTest('BatchEncoding (Python, return_tensors=TENSORFLOW)'   ):
							self.assert_dump_and_restore(
							    tokenizer_p('Small example to encode'					,			return_tensors=TensorType.TENSORFLOW   )					,			__lowercase   )

						with self.subTest('BatchEncoding (Rust, return_tensors=TENSORFLOW)'   ):
							self.assert_dump_and_restore(
							    tokenizer_r('Small example to encode'					,			return_tensors=TensorType.TENSORFLOW   )					,			__lowercase   )

					@require_torch
					@require_tokenizers
					def       lowercase_				(							self   ):







						'''simple docstring'''




						import torch

						lowerCamelCase_				:						Optional[Any]				  =				BertTokenizer.from_pretrained('bert-base-cased'   )
						lowerCamelCase_				:						Tuple				  =				BertTokenizerFast.from_pretrained('bert-base-cased'   )

						with self.subTest('BatchEncoding (Python, return_tensors=PYTORCH)'   ):
							self.assert_dump_and_restore(
							    tokenizer_p('Small example to encode'					,			return_tensors=TensorType.PYTORCH   )					,			torch.equal   )

						with self.subTest('BatchEncoding (Rust, return_tensors=PYTORCH)'   ):
							self.assert_dump_and_restore(
							    tokenizer_r('Small example to encode'					,			return_tensors=TensorType.PYTORCH   )					,			torch.equal   )

					@require_tokenizers
					def       lowercase_				(							self   ):







						'''simple docstring'''




						lowerCamelCase_				:						Dict				  =				BertTokenizer.from_pretrained('bert-base-cased'   )
						lowerCamelCase_				:						Dict				  =				BertTokenizerFast.from_pretrained('bert-base-cased'   )

						with self.subTest('Python Tokenizer'   ):
							self.assertFalse(tokenizer_p('Small example to_encode'   ).is_fast   )

						with self.subTest('Rust Tokenizer'   ):
							self.assertTrue(tokenizer_r('Small example to_encode'   ).is_fast   )

					@require_tokenizers
					def       lowercase_				(							self   ):







						'''simple docstring'''




						lowerCamelCase_				:						Union[str, Any]				  =				BertTokenizerFast.from_pretrained('bert-base-cased'   )
						lowerCamelCase_				:						Tuple				  =				tokenizer_r(['Test', '\xad', 'test']					,			is_split_into_words=__lowercase   )

						self.assertEqual(encoded.word_to_tokens(0   )					,			TokenSpan(start=1					,			end=2   )   )
						self.assertEqual(encoded.word_to_tokens(1   )					,			__lowercase   )
						self.assertEqual(encoded.word_to_tokens(2   )					,			TokenSpan(start=2					,			end=3   )   )

					def       lowercase_				(							self   ):







						'''simple docstring'''




						lowerCamelCase_				:						int				  =				BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}   )
						lowerCamelCase_				:						int				  =				batch.convert_to_tensors(tensor_type='np'   )
						self.assertEqual(tensor_batch['inputs'].shape					,			(2, 3)   )
						self.assertEqual(tensor_batch['labels'].shape					,			(2,)   )
						# test converting the converted
						with CaptureStderr() as cs:
							lowerCamelCase_				:						Union[str, Any]				  =				batch.convert_to_tensors(tensor_type='np'   )
						self.assertFalse(len(cs.err   )					,			msg=f"should have no warning, but got {cs.err}"   )

						lowerCamelCase_				:						Tuple				  =				BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}   )
						lowerCamelCase_				:						List[str]				  =				batch.convert_to_tensors(tensor_type='np'					,			prepend_batch_axis=__lowercase   )
						self.assertEqual(tensor_batch['inputs'].shape					,			(1, 3)   )
						self.assertEqual(tensor_batch['labels'].shape					,			(1,)   )

					@require_torch
					def       lowercase_				(							self   ):







						'''simple docstring'''




						lowerCamelCase_				:						int				  =				BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}   )
						lowerCamelCase_				:						List[str]				  =				batch.convert_to_tensors(tensor_type='pt'   )
						self.assertEqual(tensor_batch['inputs'].shape					,			(2, 3)   )
						self.assertEqual(tensor_batch['labels'].shape					,			(2,)   )
						# test converting the converted
						with CaptureStderr() as cs:
							lowerCamelCase_				:						List[Any]				  =				batch.convert_to_tensors(tensor_type='pt'   )
						self.assertFalse(len(cs.err   )					,			msg=f"should have no warning, but got {cs.err}"   )

						lowerCamelCase_				:						Optional[int]				  =				BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}   )
						lowerCamelCase_				:						List[str]				  =				batch.convert_to_tensors(tensor_type='pt'					,			prepend_batch_axis=__lowercase   )
						self.assertEqual(tensor_batch['inputs'].shape					,			(1, 3)   )
						self.assertEqual(tensor_batch['labels'].shape					,			(1,)   )

					@require_tf
					def       lowercase_				(							self   ):







						'''simple docstring'''




						lowerCamelCase_				:						Optional[int]				  =				BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}   )
						lowerCamelCase_				:						int				  =				batch.convert_to_tensors(tensor_type='tf'   )
						self.assertEqual(tensor_batch['inputs'].shape					,			(2, 3)   )
						self.assertEqual(tensor_batch['labels'].shape					,			(2,)   )
						# test converting the converted
						with CaptureStderr() as cs:
							lowerCamelCase_				:						Any				  =				batch.convert_to_tensors(tensor_type='tf'   )
						self.assertFalse(len(cs.err   )					,			msg=f"should have no warning, but got {cs.err}"   )

						lowerCamelCase_				:						int				  =				BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}   )
						lowerCamelCase_				:						List[Any]				  =				batch.convert_to_tensors(tensor_type='tf'					,			prepend_batch_axis=__lowercase   )
						self.assertEqual(tensor_batch['inputs'].shape					,			(1, 3)   )
						self.assertEqual(tensor_batch['labels'].shape					,			(1,)   )

					@require_flax
					def       lowercase_				(							self   ):







						'''simple docstring'''




						lowerCamelCase_				:						Dict				  =				BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}   )
						lowerCamelCase_				:						Dict				  =				batch.convert_to_tensors(tensor_type='jax'   )
						self.assertEqual(tensor_batch['inputs'].shape					,			(2, 3)   )
						self.assertEqual(tensor_batch['labels'].shape					,			(2,)   )
						# test converting the converted
						with CaptureStderr() as cs:
							lowerCamelCase_				:						Dict				  =				batch.convert_to_tensors(tensor_type='jax'   )
						self.assertFalse(len(cs.err   )					,			msg=f"should have no warning, but got {cs.err}"   )

						lowerCamelCase_				:						Dict				  =				BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}   )
						lowerCamelCase_				:						Dict				  =				batch.convert_to_tensors(tensor_type='jax'					,			prepend_batch_axis=__lowercase   )
						self.assertEqual(tensor_batch['inputs'].shape					,			(1, 3)   )
						self.assertEqual(tensor_batch['labels'].shape					,			(1,)   )

					def       lowercase_				(							self   ):







						'''simple docstring'''




						lowerCamelCase_				:						Optional[int]				  =				[{'input_ids': np.array([0, 1, 2]   )}, {'input_ids': np.array([0, 1, 2, 3]   )}]
						lowerCamelCase_				:						List[str]				  =				BertTokenizer.from_pretrained('bert-base-cased'   )

						lowerCamelCase_				:						Any				  =				tokenizer.pad(__lowercase					,			padding=__lowercase   )
						self.assertTrue(isinstance(batch['input_ids']					,			np.ndarray   )   )
						self.assertEqual(batch['input_ids'].tolist()					,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )
						lowerCamelCase_				:						Optional[int]				  =				tokenizer.pad(__lowercase					,			padding=__lowercase					,			return_tensors='np'   )
						self.assertTrue(isinstance(batch['input_ids']					,			np.ndarray   )   )
						self.assertEqual(batch['input_ids'].tolist()					,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )

					@require_torch
					def       lowercase_				(							self   ):







						'''simple docstring'''




						import torch

						lowerCamelCase_				:						Dict				  =				[{'input_ids': torch.tensor([0, 1, 2]   )}, {'input_ids': torch.tensor([0, 1, 2, 3]   )}]
						lowerCamelCase_				:						Optional[Any]				  =				BertTokenizer.from_pretrained('bert-base-cased'   )

						lowerCamelCase_				:						Dict				  =				tokenizer.pad(__lowercase					,			padding=__lowercase   )
						self.assertTrue(isinstance(batch['input_ids']					,			torch.Tensor   )   )
						self.assertEqual(batch['input_ids'].tolist()					,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )
						lowerCamelCase_				:						List[str]				  =				tokenizer.pad(__lowercase					,			padding=__lowercase					,			return_tensors='pt'   )
						self.assertTrue(isinstance(batch['input_ids']					,			torch.Tensor   )   )
						self.assertEqual(batch['input_ids'].tolist()					,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )

					@require_tf
					def       lowercase_				(							self   ):







						'''simple docstring'''




						import tensorflow as tf

						lowerCamelCase_				:						Union[str, Any]				  =				[{'input_ids': tf.constant([0, 1, 2]   )}, {'input_ids': tf.constant([0, 1, 2, 3]   )}]
						lowerCamelCase_				:						Any				  =				BertTokenizer.from_pretrained('bert-base-cased'   )

						lowerCamelCase_				:						Any				  =				tokenizer.pad(__lowercase					,			padding=__lowercase   )
						self.assertTrue(isinstance(batch['input_ids']					,			tf.Tensor   )   )
						self.assertEqual(batch['input_ids'].numpy().tolist()					,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )
						lowerCamelCase_				:						Any				  =				tokenizer.pad(__lowercase					,			padding=__lowercase					,			return_tensors='tf'   )
						self.assertTrue(isinstance(batch['input_ids']					,			tf.Tensor   )   )
						self.assertEqual(batch['input_ids'].numpy().tolist()					,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )

					@require_tokenizers
					def       lowercase_				(							self   ):







						'''simple docstring'''




						lowerCamelCase_				:						Tuple				  =				Tokenizer(WordPiece(unk_token='[UNK]'   )   )
						PreTrainedTokenizerFast(tokenizer_object=__lowercase   )





					@require_tokenizers
					def       lowercase_				(							self   ):







						'''simple docstring'''




						lowerCamelCase_				:						Optional[int]				  =				Tokenizer(WordPiece(unk_token='[UNK]'   )   )
						with tempfile.TemporaryDirectory() as tmpdirname:
							bert_tokenizer.save(os.path.join(__lowercase					,			'tokenizer.json'   )   )
							PreTrainedTokenizerFast(tokenizer_file=os.path.join(__lowercase					,			'tokenizer.json'   )   )






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCamelCase__						=		{
    'configuration_layoutlm': ['LAYOUTLM_PRETRAINED_CONFIG_ARCHIVE_MAP', 'LayoutLMConfig', 'LayoutLMOnnxConfig'],
    'tokenization_layoutlm': ['LayoutLMTokenizer'],
}

try:
							if not is_tokenizers_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							UpperCamelCase__						=		['LayoutLMTokenizerFast']

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							UpperCamelCase__						=		[
							    'LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'LayoutLMForMaskedLM',
							    'LayoutLMForSequenceClassification',
							    'LayoutLMForTokenClassification',
							    'LayoutLMForQuestionAnswering',
							    'LayoutLMModel',
							    'LayoutLMPreTrainedModel',
							]

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							UpperCamelCase__						=		[
							    'TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'TFLayoutLMForMaskedLM',
							    'TFLayoutLMForSequenceClassification',
							    'TFLayoutLMForTokenClassification',
							    'TFLayoutLMForQuestionAnswering',
							    'TFLayoutLMMainLayer',
							    'TFLayoutLMModel',
							    'TFLayoutLMPreTrainedModel',
							]


if TYPE_CHECKING:
							from .configuration_layoutlm import LAYOUTLM_PRETRAINED_CONFIG_ARCHIVE_MAP, LayoutLMConfig, LayoutLMOnnxConfig
							from .tokenization_layoutlm import LayoutLMTokenizer

							try:
														if not is_tokenizers_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_layoutlm_fast import LayoutLMTokenizerFast

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_layoutlm import (
														    LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST,
														    LayoutLMForMaskedLM,
														    LayoutLMForQuestionAnswering,
														    LayoutLMForSequenceClassification,
														    LayoutLMForTokenClassification,
														    LayoutLMModel,
														    LayoutLMPreTrainedModel,
														)
							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_layoutlm import (
														    TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFLayoutLMForMaskedLM,
														    TFLayoutLMForQuestionAnswering,
														    TFLayoutLMForSequenceClassification,
														    TFLayoutLMForTokenClassification,
														    TFLayoutLMMainLayer,
														    TFLayoutLMModel,
														    TFLayoutLMPreTrainedModel,
														)

else:
							import sys

							UpperCamelCase__						=		_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)
