


"""simple docstring"""




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
				from test_module.custom_tokenization_fast import CustomTokenizerFast
class 					__lowerCAmelCase					(   unittest.TestCase       ):


		'''simple docstring'''



		def 			__UpperCAmelCase       (  self      ):
				# A mock response for an HTTP head request to emulate server down
				__a           =			mock.Mock()
				__a           =			500
				__a           =			{}
				__a           =			HTTPError
				__a           =			{}

				# Download this model to make sure it's in the cache.
				__a           =			BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''      )

				# Under the mock environment we get a 500 error when trying to reach the tokenizer.
				with mock.patch('''requests.Session.request'''							, return_value=_a      ) as mock_head:
						__a           =			BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''      )
						# This check we did call the fake head request
						mock_head.assert_called()

		@require_tokenizers
		def 			__UpperCAmelCase       (  self      ):
				# A mock response for an HTTP head request to emulate server down
				__a           =			mock.Mock()
				__a           =			500
				__a           =			{}
				__a           =			HTTPError
				__a           =			{}

				# Download this model to make sure it's in the cache.
				__a           =			GPTaTokenizerFast.from_pretrained('''gpt2'''      )

				# Under the mock environment we get a 500 error when trying to reach the tokenizer.
				with mock.patch('''requests.Session.request'''							, return_value=_a      ) as mock_head:
						__a           =			GPTaTokenizerFast.from_pretrained('''gpt2'''      )
						# This check we did call the fake head request
						mock_head.assert_called()

		def 			__UpperCAmelCase       (  self      ):
				# This test is for deprecated behavior and can be removed in v5
				try:
						__a           =			tempfile.mktemp()
						with open(_a							, '''wb'''      ) as f:
								http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''							, _a      )

						__a           =			AlbertTokenizer.from_pretrained(_a      )
				finally:
						os.remove(_a      )

				# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
				# the current folder and have the right name.
				if os.path.isfile('''tokenizer.json'''      ):
						# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
						return
				try:
						with open('''tokenizer.json'''							, '''wb'''      ) as f:
								http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''							, _a      )
						__a           =			AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''      )
						# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
						self.assertEqual(tokenizer.vocab_size							, 1_000      )
						# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

				finally:
						os.remove('''tokenizer.json'''      )





		def 			__UpperCAmelCase       (  self      ):
				# This test is for deprecated behavior and can be removed in v5
				__a           =			AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''      )







@is_staging_test
class 					__lowerCAmelCase					(   unittest.TestCase       ):


		'''simple docstring'''



		__UpperCAmelCase		:			List[str]										=				['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']

		@classmethod
		def 			__UpperCAmelCase       (  cls      ):
				__a           =			TOKEN
				HfFolder.save_token(_a      )

		@classmethod
		def 			__UpperCAmelCase       (  cls      ):
				try:
						delete_repo(token=cls._token							, repo_id='''test-tokenizer'''      )
				except HTTPError:
						pass

				try:
						delete_repo(token=cls._token							, repo_id='''valid_org/test-tokenizer-org'''      )
				except HTTPError:
						pass

				try:
						delete_repo(token=cls._token							, repo_id='''test-dynamic-tokenizer'''      )
				except HTTPError:
						pass

		def 			__UpperCAmelCase       (  self      ):
				with tempfile.TemporaryDirectory() as tmp_dir:
						__a           =			os.path.join(_a							, '''vocab.txt'''      )
						with open(_a							, '''w'''							, encoding='''utf-8'''      ) as vocab_writer:
								vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]      )      )
						__a           =			BertTokenizer(_a      )

				tokenizer.push_to_hub('''test-tokenizer'''							, use_auth_token=self._token      )
				__a           =			BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''      )
				self.assertDictEqual(new_tokenizer.vocab							, tokenizer.vocab      )

				# Reset repo
				delete_repo(token=self._token							, repo_id='''test-tokenizer'''      )

				# Push to hub via save_pretrained
				with tempfile.TemporaryDirectory() as tmp_dir:
						tokenizer.save_pretrained(_a							, repo_id='''test-tokenizer'''							, push_to_hub=_a							, use_auth_token=self._token      )

				__a           =			BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''      )
				self.assertDictEqual(new_tokenizer.vocab							, tokenizer.vocab      )

		def 			__UpperCAmelCase       (  self      ):
				with tempfile.TemporaryDirectory() as tmp_dir:
						__a           =			os.path.join(_a							, '''vocab.txt'''      )
						with open(_a							, '''w'''							, encoding='''utf-8'''      ) as vocab_writer:
								vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]      )      )
						__a           =			BertTokenizer(_a      )

				tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''							, use_auth_token=self._token      )
				__a           =			BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''      )
				self.assertDictEqual(new_tokenizer.vocab							, tokenizer.vocab      )

				# Reset repo
				delete_repo(token=self._token							, repo_id='''valid_org/test-tokenizer-org'''      )

				# Push to hub via save_pretrained
				with tempfile.TemporaryDirectory() as tmp_dir:
						tokenizer.save_pretrained(
						    _a							, repo_id='''valid_org/test-tokenizer-org'''							, push_to_hub=_a							, use_auth_token=self._token      )

				__a           =			BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''      )
				self.assertDictEqual(new_tokenizer.vocab							, tokenizer.vocab      )





		@require_tokenizers
		def 			__UpperCAmelCase       (  self      ):
				CustomTokenizer.register_for_auto_class()
				with tempfile.TemporaryDirectory() as tmp_dir:
						__a           =			os.path.join(_a							, '''vocab.txt'''      )
						with open(_a							, '''w'''							, encoding='''utf-8'''      ) as vocab_writer:
								vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]      )      )
						__a           =			CustomTokenizer(_a      )

				# No fast custom tokenizer
				tokenizer.push_to_hub('''test-dynamic-tokenizer'''							, use_auth_token=self._token      )

				__a           =			AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''							, trust_remote_code=_a      )
				# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
				self.assertEqual(tokenizer.__class__.__name__							, '''CustomTokenizer'''      )

				# Fast and slow custom tokenizer
				CustomTokenizerFast.register_for_auto_class()
				with tempfile.TemporaryDirectory() as tmp_dir:
						__a           =			os.path.join(_a							, '''vocab.txt'''      )
						with open(_a							, '''w'''							, encoding='''utf-8'''      ) as vocab_writer:
								vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]      )      )

						__a           =			BertTokenizerFast.from_pretrained(_a      )
						bert_tokenizer.save_pretrained(_a      )
						__a           =			CustomTokenizerFast.from_pretrained(_a      )

				tokenizer.push_to_hub('''test-dynamic-tokenizer'''							, use_auth_token=self._token      )

				__a           =			AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''							, trust_remote_code=_a      )
				# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
				self.assertEqual(tokenizer.__class__.__name__							, '''CustomTokenizerFast'''      )
				__a           =			AutoTokenizer.from_pretrained(
				    f'''{USER}/test-dynamic-tokenizer'''							, use_fast=_a							, trust_remote_code=_a      )
				# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
				self.assertEqual(tokenizer.__class__.__name__							, '''CustomTokenizer'''      )







class 					__lowerCAmelCase					(   unittest.TestCase       ):


		'''simple docstring'''



		def 			__UpperCAmelCase       (  self      ):
				__a           =			Trie()
				trie.add('''Hello 友達'''      )
				self.assertEqual(trie.data							, {'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}      )
				trie.add('''Hello'''      )
				trie.data
				self.assertEqual(trie.data							, {'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}      )

		def 			__UpperCAmelCase       (  self      ):
				__a           =			Trie()
				self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''      )							, ['''[CLS] This is a extra_id_100''']      )
				trie.add('''[CLS]'''      )
				trie.add('''extra_id_1'''      )
				trie.add('''extra_id_100'''      )
				self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''      )							, ['''[CLS]''', ''' This is a ''', '''extra_id_100''']      )

		def 			__UpperCAmelCase       (  self      ):
				__a           =			Trie()
				trie.add('''A'''      )
				self.assertEqual(trie.split('''ABC'''      )							, ['''A''', '''BC''']      )
				self.assertEqual(trie.split('''BCA'''      )							, ['''BC''', '''A''']      )

		def 			__UpperCAmelCase       (  self      ):
				__a           =			Trie()
				trie.add('''TOKEN]'''      )
				trie.add('''[SPECIAL_TOKEN]'''      )
				self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''      )							, ['''This is something ''', '''[SPECIAL_TOKEN]''']      )

		def 			__UpperCAmelCase       (  self      ):
				__a           =			Trie()
				trie.add('''A'''      )
				trie.add('''P'''      )
				trie.add('''[SPECIAL_TOKEN]'''      )
				self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''      )							, ['''This is something ''', '''[SPECIAL_TOKEN]''']      )

		def 			__UpperCAmelCase       (  self      ):
				__a           =			Trie()
				trie.add('''AB'''      )
				trie.add('''B'''      )
				trie.add('''C'''      )
				self.assertEqual(trie.split('''ABC'''      )							, ['''AB''', '''C''']      )

		def 			__UpperCAmelCase       (  self      ):
				__a           =			Trie()
				trie.add('''ABC'''      )
				trie.add('''B'''      )
				trie.add('''CD'''      )
				self.assertEqual(trie.split('''ABCD'''      )							, ['''ABC''', '''D''']      )





		def 			__UpperCAmelCase       (  self      ):
				# Even if the offsets are wrong, we necessarily output correct string
				# parts.
				__a           =			Trie()
				__a           =			trie.cut_text('''ABC'''							, [0, 0, 2, 1, 2, 3]      )
				self.assertEqual(_a							, ['''AB''', '''C''']      )










"""simple docstring"""




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowercase_			     =  {"configuration_xglm": ["XGLM_PRETRAINED_CONFIG_ARCHIVE_MAP", "XGLMConfig"]}

try:
				if not is_sentencepiece_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_			     =  ["XGLMTokenizer"]

try:
				if not is_tokenizers_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_			     =  ["XGLMTokenizerFast"]

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_			     =  [
				    "XGLM_PRETRAINED_MODEL_ARCHIVE_LIST",
				    "XGLMForCausalLM",
				    "XGLMModel",
				    "XGLMPreTrainedModel",
				]


try:
				if not is_flax_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_			     =  [
				    "FlaxXGLMForCausalLM",
				    "FlaxXGLMModel",
				    "FlaxXGLMPreTrainedModel",
				]


try:
				if not is_tf_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_			     =  [
				    "TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST",
				    "TFXGLMForCausalLM",
				    "TFXGLMModel",
				    "TFXGLMPreTrainedModel",
				]


if TYPE_CHECKING:
				from .configuration_xglm import XGLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XGLMConfig

				try:
								if not is_sentencepiece_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_xglm import XGLMTokenizer

				try:
								if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_xglm_fast import XGLMTokenizerFast

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_xglm import XGLM_PRETRAINED_MODEL_ARCHIVE_LIST, XGLMForCausalLM, XGLMModel, XGLMPreTrainedModel

				try:
								if not is_flax_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_flax_xglm import FlaxXGLMForCausalLM, FlaxXGLMModel, FlaxXGLMPreTrainedModel

				try:
								if not is_tf_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_tf_xglm import (
								    TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST,
								    TFXGLMForCausalLM,
								    TFXGLMModel,
								    TFXGLMPreTrainedModel,
								)


else:
				import sys

				lowercase_			     =  _LazyModule(__name__, globals()["__file__"], _import_structure)






