

'''simple docstring'''

import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
				from test_module.custom_tokenization_fast import CustomTokenizerFast







class   UpperCamelCase__    (   unittest.TestCase   ):


		"""simple docstring"""

		def       A_       ( self			):


							'''simple docstring'''

							UpperCAmelCase		:							Any   					=   mock.Mock()
							UpperCAmelCase		:							Union[str, Any]   					=   5_0_0
							UpperCAmelCase		:							Union[str, Any]   					=   {}
							UpperCAmelCase		:							Union[str, Any]   					=   HTTPError
							UpperCAmelCase		:							Union[str, Any]   					=   {}

							# Download this model to make sure it's in the cache.
							UpperCAmelCase		:							Dict   					=   BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"			)

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch("requests.Session.request"    ,			return_value=lowerCAmelCase_			) as mock_head:
												UpperCAmelCase		:							Optional[int]   					=   BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"			)
												# This check we did call the fake head request
												mock_head.assert_called()







		@require_tokenizers
		def       A_       ( self			):


							'''simple docstring'''

							UpperCAmelCase		:							Optional[Any]   					=   mock.Mock()
							UpperCAmelCase		:							int   					=   5_0_0
							UpperCAmelCase		:							Tuple   					=   {}
							UpperCAmelCase		:							Tuple   					=   HTTPError
							UpperCAmelCase		:							Optional[int]   					=   {}

							# Download this model to make sure it's in the cache.
							UpperCAmelCase		:							Any   					=   GPTaTokenizerFast.from_pretrained("gpt2"			)

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch("requests.Session.request"    ,			return_value=lowerCAmelCase_			) as mock_head:
												UpperCAmelCase		:							Any   					=   GPTaTokenizerFast.from_pretrained("gpt2"			)
												# This check we did call the fake head request
												mock_head.assert_called()







		def       A_       ( self			):


							'''simple docstring'''

							try:
												UpperCAmelCase		:							List[Any]   					=   tempfile.mktemp()
												with open(lowerCAmelCase_    ,			"wb"			) as f:
																	http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"    ,			lowerCAmelCase_			)

												UpperCAmelCase		:							str   					=   AlbertTokenizer.from_pretrained(lowerCAmelCase_			)
							finally:
												os.remove(lowerCAmelCase_			)

							# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
							# the current folder and have the right name.
							if os.path.isfile("tokenizer.json"			):
												# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
												return
							try:
												with open("tokenizer.json"    ,			"wb"			) as f:
																	http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"    ,			lowerCAmelCase_			)
												UpperCAmelCase		:							List[Any]   					=   AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"			)
												# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
												self.assertEqual(tokenizer.vocab_size    ,			1_0_0_0			)
												# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

							finally:
												os.remove("tokenizer.json"			)







		def       A_       ( self			):


							'''simple docstring'''

							UpperCAmelCase		:							Dict   					=   AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"			)







@is_staging_test
class   UpperCamelCase__    (   unittest.TestCase   ):


		"""simple docstring"""

		SCREAMING_SNAKE_CASE__  :	List[Any]   =							["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]







		@classmethod
		def       A_       ( cls			):


							'''simple docstring'''

							UpperCAmelCase		:							Optional[int]   					=   TOKEN
							HfFolder.save_token(lowerCAmelCase_			)







		@classmethod
		def       A_       ( cls			):


							'''simple docstring'''

							try:
												delete_repo(token=cls._token    ,			repo_id="test-tokenizer"			)
							except HTTPError:
												pass

							try:
												delete_repo(token=cls._token    ,			repo_id="valid_org/test-tokenizer-org"			)
							except HTTPError:
												pass

							try:
												delete_repo(token=cls._token    ,			repo_id="test-dynamic-tokenizer"			)
							except HTTPError:
												pass







		def       A_       ( self			):


							'''simple docstring'''

							with tempfile.TemporaryDirectory() as tmp_dir:
												UpperCAmelCase		:							List[Any]   					=   os.path.join(lowerCAmelCase_    ,			"vocab.txt"			)
												with open(lowerCAmelCase_    ,			"w"    ,			encoding="utf-8"			) as vocab_writer:
																	vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]			)			)
												UpperCAmelCase		:							List[Any]   					=   BertTokenizer(lowerCAmelCase_			)

							tokenizer.push_to_hub("test-tokenizer"    ,			use_auth_token=self._token			)
							UpperCAmelCase		:							Any   					=   BertTokenizer.from_pretrained(f"{USER}/test-tokenizer"			)
							self.assertDictEqual(new_tokenizer.vocab    ,			tokenizer.vocab			)

							# Reset repo
							delete_repo(token=self._token    ,			repo_id="test-tokenizer"			)

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
												tokenizer.save_pretrained(lowerCAmelCase_    ,			repo_id="test-tokenizer"    ,			push_to_hub=lowerCAmelCase_    ,			use_auth_token=self._token			)

							UpperCAmelCase		:							Optional[Any]   					=   BertTokenizer.from_pretrained(f"{USER}/test-tokenizer"			)
							self.assertDictEqual(new_tokenizer.vocab    ,			tokenizer.vocab			)







		def       A_       ( self			):


							'''simple docstring'''

							with tempfile.TemporaryDirectory() as tmp_dir:
												UpperCAmelCase		:							Optional[Any]   					=   os.path.join(lowerCAmelCase_    ,			"vocab.txt"			)
												with open(lowerCAmelCase_    ,			"w"    ,			encoding="utf-8"			) as vocab_writer:
																	vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]			)			)
												UpperCAmelCase		:							Union[str, Any]   					=   BertTokenizer(lowerCAmelCase_			)

							tokenizer.push_to_hub("valid_org/test-tokenizer-org"    ,			use_auth_token=self._token			)
							UpperCAmelCase		:							str   					=   BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"			)
							self.assertDictEqual(new_tokenizer.vocab    ,			tokenizer.vocab			)

							# Reset repo
							delete_repo(token=self._token    ,			repo_id="valid_org/test-tokenizer-org"			)

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
												tokenizer.save_pretrained(
												    lowerCAmelCase_    ,			repo_id="valid_org/test-tokenizer-org"    ,			push_to_hub=lowerCAmelCase_    ,			use_auth_token=self._token			)

							UpperCAmelCase		:							int   					=   BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"			)
							self.assertDictEqual(new_tokenizer.vocab    ,			tokenizer.vocab			)







		@require_tokenizers
		def       A_       ( self			):


							'''simple docstring'''

							CustomTokenizer.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
												UpperCAmelCase		:							Any   					=   os.path.join(lowerCAmelCase_    ,			"vocab.txt"			)
												with open(lowerCAmelCase_    ,			"w"    ,			encoding="utf-8"			) as vocab_writer:
																	vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]			)			)
												UpperCAmelCase		:							int   					=   CustomTokenizer(lowerCAmelCase_			)

							# No fast custom tokenizer
							tokenizer.push_to_hub("test-dynamic-tokenizer"    ,			use_auth_token=self._token			)

							UpperCAmelCase		:							Any   					=   AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer"    ,			trust_remote_code=lowerCAmelCase_			)
							# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__    ,			"CustomTokenizer"			)

							# Fast and slow custom tokenizer
							CustomTokenizerFast.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
												UpperCAmelCase		:							Tuple   					=   os.path.join(lowerCAmelCase_    ,			"vocab.txt"			)
												with open(lowerCAmelCase_    ,			"w"    ,			encoding="utf-8"			) as vocab_writer:
																	vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]			)			)

												UpperCAmelCase		:							List[str]   					=   BertTokenizerFast.from_pretrained(lowerCAmelCase_			)
												bert_tokenizer.save_pretrained(lowerCAmelCase_			)
												UpperCAmelCase		:							str   					=   CustomTokenizerFast.from_pretrained(lowerCAmelCase_			)

							tokenizer.push_to_hub("test-dynamic-tokenizer"    ,			use_auth_token=self._token			)

							UpperCAmelCase		:							str   					=   AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer"    ,			trust_remote_code=lowerCAmelCase_			)
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__    ,			"CustomTokenizerFast"			)
							UpperCAmelCase		:							List[Any]   					=   AutoTokenizer.from_pretrained(
							    f"{USER}/test-dynamic-tokenizer"    ,			use_fast=lowerCAmelCase_    ,			trust_remote_code=lowerCAmelCase_			)
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__    ,			"CustomTokenizer"			)







class   UpperCamelCase__    (   unittest.TestCase   ):


		"""simple docstring"""

		def       A_       ( self			):


							'''simple docstring'''

							UpperCAmelCase		:							Optional[int]   					=   Trie()
							trie.add("Hello 友達"			)
							self.assertEqual(trie.data    ,			{"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}			)
							trie.add("Hello"			)
							trie.data
							self.assertEqual(trie.data    ,			{"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}			)







		def       A_       ( self			):


							'''simple docstring'''

							UpperCAmelCase		:							int   					=   Trie()
							self.assertEqual(trie.split("[CLS] This is a extra_id_100"			)    ,			["[CLS] This is a extra_id_100"]			)
							trie.add("[CLS]"			)
							trie.add("extra_id_1"			)
							trie.add("extra_id_100"			)
							self.assertEqual(trie.split("[CLS] This is a extra_id_100"			)    ,			["[CLS]", " This is a ", "extra_id_100"]			)







		def       A_       ( self			):


							'''simple docstring'''

							UpperCAmelCase		:							List[str]   					=   Trie()
							trie.add("A"			)
							self.assertEqual(trie.split("ABC"			)    ,			["A", "BC"]			)
							self.assertEqual(trie.split("BCA"			)    ,			["BC", "A"]			)







		def       A_       ( self			):


							'''simple docstring'''

							UpperCAmelCase		:							List[str]   					=   Trie()
							trie.add("TOKEN]"			)
							trie.add("[SPECIAL_TOKEN]"			)
							self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"			)    ,			["This is something ", "[SPECIAL_TOKEN]"]			)







		def       A_       ( self			):


							'''simple docstring'''

							UpperCAmelCase		:							int   					=   Trie()
							trie.add("A"			)
							trie.add("P"			)
							trie.add("[SPECIAL_TOKEN]"			)
							self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"			)    ,			["This is something ", "[SPECIAL_TOKEN]"]			)







		def       A_       ( self			):


							'''simple docstring'''

							UpperCAmelCase		:							str   					=   Trie()
							trie.add("AB"			)
							trie.add("B"			)
							trie.add("C"			)
							self.assertEqual(trie.split("ABC"			)    ,			["AB", "C"]			)







		def       A_       ( self			):


							'''simple docstring'''

							UpperCAmelCase		:							int   					=   Trie()
							trie.add("ABC"			)
							trie.add("B"			)
							trie.add("CD"			)
							self.assertEqual(trie.split("ABCD"			)    ,			["ABC", "D"]			)







		def       A_       ( self			):


							'''simple docstring'''

							UpperCAmelCase		:							Optional[int]   					=   Trie()
							UpperCAmelCase		:							str   					=   trie.cut_text("ABC"    ,			[0, 0, 2, 1, 2, 3]			)
							self.assertEqual(lowerCAmelCase_    ,			["AB", "C"]			)


'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


lowercase    :							Any      =					{
    "configuration_chinese_clip": [
        "CHINESE_CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "ChineseCLIPConfig",
        "ChineseCLIPOnnxConfig",
        "ChineseCLIPTextConfig",
        "ChineseCLIPVisionConfig",
    ],
    "processing_chinese_clip": ["ChineseCLIPProcessor"],
}

try:
					if not is_vision_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowercase    :							Optional[Any]      =					["ChineseCLIPFeatureExtractor"]
					lowercase    :							List[Any]      =					["ChineseCLIPImageProcessor"]

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowercase    :							Any      =					[
					    "CHINESE_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "ChineseCLIPModel",
					    "ChineseCLIPPreTrainedModel",
					    "ChineseCLIPTextModel",
					    "ChineseCLIPVisionModel",
					]

if TYPE_CHECKING:
					from .configuration_chinese_clip import (
					    CHINESE_CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
					    ChineseCLIPConfig,
					    ChineseCLIPOnnxConfig,
					    ChineseCLIPTextConfig,
					    ChineseCLIPVisionConfig,
					)
					from .processing_chinese_clip import ChineseCLIPProcessor

					try:
										if not is_vision_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .feature_extraction_chinese_clip import ChineseCLIPFeatureExtractor, ChineseCLIPImageProcessor

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_chinese_clip import (
										    CHINESE_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
										    ChineseCLIPModel,
										    ChineseCLIPPreTrainedModel,
										    ChineseCLIPTextModel,
										    ChineseCLIPVisionModel,
										)

else:
					import sys

					lowercase    :							List[Any]      =					_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)

