




'''simple docstring'''







import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
						from tokenizers import Tokenizer
						from tokenizers.models import WordPiece





class 	lowerCamelCase_				(			unittest.TestCase				):




					'''simple docstring'''






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase					):
							_snake_case      			=			list(tokenizer_class.max_model_input_sizes.keys()					)
							for model_name in sa_models[:1]:
									_snake_case      			=			tokenizer_class.from_pretrained(__UpperCamelCase					)
									self.assertIsNotNone(__UpperCamelCase					)
									self.assertIsInstance(__UpperCamelCase,    __UpperCamelCase					)
									self.assertIsInstance(__UpperCamelCase,    __UpperCamelCase					)

									for special_tok in tokenizer.all_special_tokens:
											self.assertIsInstance(__UpperCamelCase,    __UpperCamelCase					)
											_snake_case      			=			tokenizer.convert_tokens_to_ids(__UpperCamelCase					)
											self.assertIsInstance(__UpperCamelCase,    __UpperCamelCase					)






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase,    __UpperCamelCase = None					):
							_snake_case      			=			pickle.dumps(__UpperCamelCase					)
							self.assertIsNotNone(__UpperCamelCase					)

							_snake_case      			=			pickle.loads(__UpperCamelCase					)

							# Ensure is_fast is correctly restored
							self.assertEqual(be_restored.is_fast,    be_original.is_fast					)

							# Ensure encodings are potentially correctly restored
							if be_original.is_fast:
									self.assertIsNotNone(be_restored.encodings					)
							else:
									self.assertIsNone(be_restored.encodings					)

							# Ensure the keys are the same
							for original_v, restored_v in zip(be_original.values(),    be_restored.values()					):
									if equal_op:
											self.assertTrue(equal_op(__UpperCamelCase,    __UpperCamelCase					)					)
									else:
											self.assertEqual(__UpperCamelCase,    __UpperCamelCase					)






					@slow
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							self.check_tokenizer_from_pretrained(__UpperCamelCase					)






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							self.assertEqual(TensorType("tf"					),    TensorType.TENSORFLOW					)
							self.assertEqual(TensorType("pt"					),    TensorType.PYTORCH					)
							self.assertEqual(TensorType("np"					),    TensorType.NUMPY					)






					@require_tokenizers
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							import numpy as np

							_snake_case      			=			BertTokenizer.from_pretrained("bert-base-cased"					)
							_snake_case      			=			BertTokenizerFast.from_pretrained("bert-base-cased"					)

							# Python no tensor
							with self.subTest("BatchEncoding (Python, return_tensors=None)"					):
									self.assert_dump_and_restore(tokenizer_p("Small example to encode"					)					)

							with self.subTest("BatchEncoding (Python, return_tensors=NUMPY)"					):
									self.assert_dump_and_restore(
									    tokenizer_p("Small example to encode",    return_tensors=TensorType.NUMPY					),    np.array_equal					)

							with self.subTest("BatchEncoding (Rust, return_tensors=None)"					):
									self.assert_dump_and_restore(tokenizer_r("Small example to encode"					)					)

							with self.subTest("BatchEncoding (Rust, return_tensors=NUMPY)"					):
									self.assert_dump_and_restore(
									    tokenizer_r("Small example to encode",    return_tensors=TensorType.NUMPY					),    np.array_equal					)






					@require_tf
					@require_tokenizers
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							import tensorflow as tf

							def tf_array_equals(__UpperCamelCase,    __UpperCamelCase					):
									return tf.reduce_all(tf.equal(__UpperCamelCase,    __UpperCamelCase					)					)

							_snake_case      			=			BertTokenizer.from_pretrained("bert-base-cased"					)
							_snake_case      			=			BertTokenizerFast.from_pretrained("bert-base-cased"					)

							with self.subTest("BatchEncoding (Python, return_tensors=TENSORFLOW)"					):
									self.assert_dump_and_restore(
									    tokenizer_p("Small example to encode",    return_tensors=TensorType.TENSORFLOW					),    __UpperCamelCase					)

							with self.subTest("BatchEncoding (Rust, return_tensors=TENSORFLOW)"					):
									self.assert_dump_and_restore(
									    tokenizer_r("Small example to encode",    return_tensors=TensorType.TENSORFLOW					),    __UpperCamelCase					)






					@require_torch
					@require_tokenizers
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							import torch

							_snake_case      			=			BertTokenizer.from_pretrained("bert-base-cased"					)
							_snake_case      			=			BertTokenizerFast.from_pretrained("bert-base-cased"					)

							with self.subTest("BatchEncoding (Python, return_tensors=PYTORCH)"					):
									self.assert_dump_and_restore(
									    tokenizer_p("Small example to encode",    return_tensors=TensorType.PYTORCH					),    torch.equal					)

							with self.subTest("BatchEncoding (Rust, return_tensors=PYTORCH)"					):
									self.assert_dump_and_restore(
									    tokenizer_r("Small example to encode",    return_tensors=TensorType.PYTORCH					),    torch.equal					)






					@require_tokenizers
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			BertTokenizer.from_pretrained("bert-base-cased"					)
							_snake_case      			=			BertTokenizerFast.from_pretrained("bert-base-cased"					)

							with self.subTest("Python Tokenizer"					):
									self.assertFalse(tokenizer_p("Small example to_encode"					).is_fast					)

							with self.subTest("Rust Tokenizer"					):
									self.assertTrue(tokenizer_r("Small example to_encode"					).is_fast					)






					@require_tokenizers
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			BertTokenizerFast.from_pretrained("bert-base-cased"					)
							_snake_case      			=			tokenizer_r(["Test", "\xad", "test"],    is_split_into_words=__UpperCamelCase					)

							self.assertEqual(encoded.word_to_tokens(0					),    TokenSpan(start=1,    end=2					)					)
							self.assertEqual(encoded.word_to_tokens(1					),    __UpperCamelCase					)
							self.assertEqual(encoded.word_to_tokens(2					),    TokenSpan(start=2,    end=3					)					)






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}					)
							_snake_case      			=			batch.convert_to_tensors(tensor_type="np"					)
							self.assertEqual(tensor_batch["inputs"].shape,    (2, 3)					)
							self.assertEqual(tensor_batch["labels"].shape,    (2,)					)
							# test converting the converted
							with CaptureStderr() as cs:
									_snake_case      			=			batch.convert_to_tensors(tensor_type="np"					)
							self.assertFalse(len(cs.err					),    msg=f'''should have no warning, but got {cs.err}'''					)

							_snake_case      			=			BatchEncoding({"inputs": [1, 2, 3], "labels": 0}					)
							_snake_case      			=			batch.convert_to_tensors(tensor_type="np",    prepend_batch_axis=__UpperCamelCase					)
							self.assertEqual(tensor_batch["inputs"].shape,    (1, 3)					)
							self.assertEqual(tensor_batch["labels"].shape,    (1,)					)






					@require_torch
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}					)
							_snake_case      			=			batch.convert_to_tensors(tensor_type="pt"					)
							self.assertEqual(tensor_batch["inputs"].shape,    (2, 3)					)
							self.assertEqual(tensor_batch["labels"].shape,    (2,)					)
							# test converting the converted
							with CaptureStderr() as cs:
									_snake_case      			=			batch.convert_to_tensors(tensor_type="pt"					)
							self.assertFalse(len(cs.err					),    msg=f'''should have no warning, but got {cs.err}'''					)

							_snake_case      			=			BatchEncoding({"inputs": [1, 2, 3], "labels": 0}					)
							_snake_case      			=			batch.convert_to_tensors(tensor_type="pt",    prepend_batch_axis=__UpperCamelCase					)
							self.assertEqual(tensor_batch["inputs"].shape,    (1, 3)					)
							self.assertEqual(tensor_batch["labels"].shape,    (1,)					)






					@require_tf
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}					)
							_snake_case      			=			batch.convert_to_tensors(tensor_type="tf"					)
							self.assertEqual(tensor_batch["inputs"].shape,    (2, 3)					)
							self.assertEqual(tensor_batch["labels"].shape,    (2,)					)
							# test converting the converted
							with CaptureStderr() as cs:
									_snake_case      			=			batch.convert_to_tensors(tensor_type="tf"					)
							self.assertFalse(len(cs.err					),    msg=f'''should have no warning, but got {cs.err}'''					)

							_snake_case      			=			BatchEncoding({"inputs": [1, 2, 3], "labels": 0}					)
							_snake_case      			=			batch.convert_to_tensors(tensor_type="tf",    prepend_batch_axis=__UpperCamelCase					)
							self.assertEqual(tensor_batch["inputs"].shape,    (1, 3)					)
							self.assertEqual(tensor_batch["labels"].shape,    (1,)					)






					@require_flax
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}					)
							_snake_case      			=			batch.convert_to_tensors(tensor_type="jax"					)
							self.assertEqual(tensor_batch["inputs"].shape,    (2, 3)					)
							self.assertEqual(tensor_batch["labels"].shape,    (2,)					)
							# test converting the converted
							with CaptureStderr() as cs:
									_snake_case      			=			batch.convert_to_tensors(tensor_type="jax"					)
							self.assertFalse(len(cs.err					),    msg=f'''should have no warning, but got {cs.err}'''					)

							_snake_case      			=			BatchEncoding({"inputs": [1, 2, 3], "labels": 0}					)
							_snake_case      			=			batch.convert_to_tensors(tensor_type="jax",    prepend_batch_axis=__UpperCamelCase					)
							self.assertEqual(tensor_batch["inputs"].shape,    (1, 3)					)
							self.assertEqual(tensor_batch["labels"].shape,    (1,)					)






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			[{"input_ids": np.array([0, 1, 2]					)}, {"input_ids": np.array([0, 1, 2, 3]					)}]
							_snake_case      			=			BertTokenizer.from_pretrained("bert-base-cased"					)

							_snake_case      			=			tokenizer.pad(__UpperCamelCase,    padding=__UpperCamelCase					)
							self.assertTrue(isinstance(batch["input_ids"],    np.ndarray					)					)
							self.assertEqual(batch["input_ids"].tolist(),    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
							_snake_case      			=			tokenizer.pad(__UpperCamelCase,    padding=__UpperCamelCase,    return_tensors="np"					)
							self.assertTrue(isinstance(batch["input_ids"],    np.ndarray					)					)
							self.assertEqual(batch["input_ids"].tolist(),    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)






					@require_torch
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							import torch

							_snake_case      			=			[{"input_ids": torch.tensor([0, 1, 2]					)}, {"input_ids": torch.tensor([0, 1, 2, 3]					)}]
							_snake_case      			=			BertTokenizer.from_pretrained("bert-base-cased"					)

							_snake_case      			=			tokenizer.pad(__UpperCamelCase,    padding=__UpperCamelCase					)
							self.assertTrue(isinstance(batch["input_ids"],    torch.Tensor					)					)
							self.assertEqual(batch["input_ids"].tolist(),    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
							_snake_case      			=			tokenizer.pad(__UpperCamelCase,    padding=__UpperCamelCase,    return_tensors="pt"					)
							self.assertTrue(isinstance(batch["input_ids"],    torch.Tensor					)					)
							self.assertEqual(batch["input_ids"].tolist(),    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)






					@require_tf
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							import tensorflow as tf

							_snake_case      			=			[{"input_ids": tf.constant([0, 1, 2]					)}, {"input_ids": tf.constant([0, 1, 2, 3]					)}]
							_snake_case      			=			BertTokenizer.from_pretrained("bert-base-cased"					)

							_snake_case      			=			tokenizer.pad(__UpperCamelCase,    padding=__UpperCamelCase					)
							self.assertTrue(isinstance(batch["input_ids"],    tf.Tensor					)					)
							self.assertEqual(batch["input_ids"].numpy().tolist(),    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
							_snake_case      			=			tokenizer.pad(__UpperCamelCase,    padding=__UpperCamelCase,    return_tensors="tf"					)
							self.assertTrue(isinstance(batch["input_ids"],    tf.Tensor					)					)
							self.assertEqual(batch["input_ids"].numpy().tolist(),    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)






					@require_tokenizers
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			Tokenizer(WordPiece(unk_token="[UNK]"					)					)
							PreTrainedTokenizerFast(tokenizer_object=__UpperCamelCase					)






					@require_tokenizers
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			Tokenizer(WordPiece(unk_token="[UNK]"					)					)
							with tempfile.TemporaryDirectory() as tmpdirname:
									bert_tokenizer.save(os.path.join(__UpperCamelCase,    "tokenizer.json"					)					)
									PreTrainedTokenizerFast(tokenizer_file=os.path.join(__UpperCamelCase,    "tokenizer.json"					)					)











'''simple docstring'''







import enum
import inspect
from typing import Iterable, List, Optional, Tuple, Union





class 	lowerCamelCase_				(			enum.Enum				):




					'''simple docstring'''






					lowercase__       =					"""timm"""
					lowercase__       =					"""transformers"""






def 						_lowercase						(      lowercase_		,				lowercase_		,				lowercase_    ):



		'''simple docstring'''







		if stage_names is None:
				raise ValueError("Stage_names must be set for transformers backbones"    )

		if out_features is not None:
				if not isinstance(lowercase_		,				(list,)    ):
						raise ValueError(f'''out_features must be a list {type(lowercase_    )}'''    )
				if any(feat not in stage_names for feat in out_features    ):
						raise ValueError(f'''out_features must be a subset of stage_names: {stage_names} got {out_features}'''    )

		if out_indices is not None:
				if not isinstance(lowercase_		,				(list, tuple)    ):
						raise ValueError(f'''out_indices must be a list or tuple, got {type(lowercase_    )}'''    )
				if any(idx >= len(lowercase_    ) for idx in out_indices    ):
						raise ValueError("out_indices must be valid indices for stage_names {stage_names}, got {out_indices}"    )

		if out_features is not None and out_indices is not None:
				if len(lowercase_    ) != len(lowercase_    ):
						raise ValueError("out_features and out_indices should have the same length if both are set"    )
				if out_features != [stage_names[idx] for idx in out_indices]:
						raise ValueError("out_features and out_indices should correspond to the same stages if both are set"    )






def 						_lowercase						(      lowercase_		,				lowercase_		,				lowercase_		,				):



		'''simple docstring'''







		if out_indices is None and out_features is None:
				_snake_case      			=			[len(lowercase_    ) - 1]
				_snake_case      			=			[stage_names[-1]]
		elif out_indices is None and out_features is not None:
				_snake_case      			=			[stage_names.index(lowercase_    ) for layer in out_features]
		elif out_features is None and out_indices is not None:
				_snake_case      			=			[stage_names[idx] for idx in out_indices]
		return out_features, out_indices






def 						_lowercase						(      lowercase_		,				lowercase_		,				lowercase_		,				):



		'''simple docstring'''







		verify_out_features_out_indices(out_features=lowercase_		,				out_indices=lowercase_		,				stage_names=lowercase_    )
		_snake_case			,       _snake_case      			=			_align_output_features_output_indices(
		    out_features=lowercase_		,				out_indices=lowercase_		,				stage_names=lowercase_    )
		# Verify that the aligned out_features and out_indices are valid
		verify_out_features_out_indices(out_features=lowercase_		,				out_indices=lowercase_		,				stage_names=lowercase_    )
		return output_features, output_indices





class 	lowerCamelCase_				:




					'''simple docstring'''






					lowercase__       =					None






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase					):
							if getattr(self,    "_backbone",    __UpperCamelCase					) is None:
									raise ValueError("self._backbone must be set before calling _init_timm_backbone"					)

							# These will diagree with the defaults for the transformers models e.g. for resnet50
							# the transformer model has out_features = ['stem', 'stage1', 'stage2', 'stage3', 'stage4']
							# the timm model has out_features = ['act', 'layer1', 'layer2', 'layer3', 'layer4']
							_snake_case      			=			[stage["module"] for stage in self._backbone.feature_info.info]
							_snake_case      			=			[stage["num_chs"] for stage in self._backbone.feature_info.info]
							_snake_case      			=			self._backbone.feature_info.out_indices
							_snake_case      			=			self._backbone.feature_info.module_name()

							# We verify the out indices and out features are valid
							verify_out_features_out_indices(
							    out_features=__UpperCamelCase,    out_indices=__UpperCamelCase,    stage_names=self.stage_names					)
							_snake_case			,       _snake_case      			=			out_features, out_indices






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase					):
							_snake_case      			=			getattr(__UpperCamelCase,    "stage_names"					)
							_snake_case      			=			getattr(__UpperCamelCase,    "out_features",    __UpperCamelCase					)
							_snake_case      			=			getattr(__UpperCamelCase,    "out_indices",    __UpperCamelCase					)

							_snake_case      			=			stage_names
							_snake_case			,       _snake_case      			=			get_aligned_output_features_output_indices(
							    out_features=__UpperCamelCase,    out_indices=__UpperCamelCase,    stage_names=__UpperCamelCase					)
							# Number of channels for each stage. This is set in the transformer backbone model init
							_snake_case      			=			None






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase					):
							_snake_case      			=			config

							_snake_case      			=			getattr(__UpperCamelCase,    "use_timm_backbone",    __UpperCamelCase					)
							_snake_case      			=			BackboneType.TIMM if self.use_timm_backbone else BackboneType.TRANSFORMERS

							if self.backbone_type == BackboneType.TIMM:
									self._init_timm_backbone(__UpperCamelCase					)
							elif self.backbone_type == BackboneType.TRANSFORMERS:
									self._init_transformers_backbone(__UpperCamelCase					)
							else:
									raise ValueError(f'''backbone_type {self.backbone_type} not supported.'''					)






					@property
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							return self._out_features






					@out_features.setter
					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase					):
							_snake_case			,       _snake_case      			=			get_aligned_output_features_output_indices(
							    out_features=__UpperCamelCase,    out_indices=__UpperCamelCase,    stage_names=self.stage_names					)






					@property
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							return self._out_indices






					@out_indices.setter
					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase					):
							_snake_case			,       _snake_case      			=			get_aligned_output_features_output_indices(
							    out_features=__UpperCamelCase,    out_indices=__UpperCamelCase,    stage_names=self.stage_names					)






					@property
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							# the current backbones will output the number of channels for each stage
							# even if that stage is not in the out_features list.
							return {stage: self.num_features[i] for i, stage in enumerate(self.stage_names					)}






					@property
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							return [self.out_feature_channels[name] for name in self.out_features]






					def 							__SCREAMING_SNAKE_CASE      (       self,    *__UpperCamelCase,    **__UpperCamelCase					):
							_snake_case      			=			dict(inspect.signature(self.forward					).parameters					)
							_snake_case      			=			{k: v for k, v in kwargs.items() if k in signature}
							return self(*__UpperCamelCase,    **__UpperCamelCase					)






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase,    __UpperCamelCase = None,    __UpperCamelCase = None,    __UpperCamelCase = None,    ):
							raise NotImplementedError("This method should be implemented by the derived class."					)






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			super().to_dict()
							_snake_case      			=			output.pop("_out_features"					)
							_snake_case      			=			output.pop("_out_indices"					)
							return output





class 	lowerCamelCase_				:




					'''simple docstring'''






					@property
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							return self._out_features






					@out_features.setter
					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase					):
							_snake_case			,       _snake_case      			=			get_aligned_output_features_output_indices(
							    out_features=__UpperCamelCase,    out_indices=__UpperCamelCase,    stage_names=self.stage_names					)






					@property
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							return self._out_indices






					@out_indices.setter
					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase					):
							_snake_case			,       _snake_case      			=			get_aligned_output_features_output_indices(
							    out_features=__UpperCamelCase,    out_indices=__UpperCamelCase,    stage_names=self.stage_names					)






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			super().to_dict()
							_snake_case      			=			output.pop("_out_features"					)
							_snake_case      			=			output.pop("_out_indices"					)
							return output





