
'''simple docstring'''




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
      from test_module.custom_tokenization_fast import CustomTokenizerFast





class  A				(							unittest.TestCase						):





     def 				__lowerCAmelCase							(							self	) ->				Union[str, Any]:






         """simple docstring"""






         A       :   Dict					=						mock.Mock()
         A       :   str					=						500
         A       :   List[str]					=						{}
         A       :   List[Any]					=						HTTPError
         A       :   str					=						{}

         # Download this model to make sure it's in the cache.
         A       :   Tuple					=						BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''	)

         # Under the mock environment we get a 500 error when trying to reach the tokenizer.
         with mock.patch('''requests.Session.request'''						,			return_value=SCREAMING_SNAKE_CASE	) as mock_head:
             A       :   Dict					=						BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''	)
             # This check we did call the fake head request
             mock_head.assert_called()





     @require_tokenizers
     def 				__lowerCAmelCase							(							self	) ->				Dict:






         """simple docstring"""






         A       :   str					=						mock.Mock()
         A       :   Optional[Any]					=						500
         A       :   str					=						{}
         A       :   Tuple					=						HTTPError
         A       :   str					=						{}

         # Download this model to make sure it's in the cache.
         A       :   str					=						GPTaTokenizerFast.from_pretrained('''gpt2'''	)

         # Under the mock environment we get a 500 error when trying to reach the tokenizer.
         with mock.patch('''requests.Session.request'''						,			return_value=SCREAMING_SNAKE_CASE	) as mock_head:
             A       :   Union[str, Any]					=						GPTaTokenizerFast.from_pretrained('''gpt2'''	)
             # This check we did call the fake head request
             mock_head.assert_called()





     def 				__lowerCAmelCase							(							self	) ->				Any:






         """simple docstring"""






         try:
             A       :   Dict					=						tempfile.mktemp()
             with open(SCREAMING_SNAKE_CASE						,			'''wb'''	) as f:
                 http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''						,			SCREAMING_SNAKE_CASE	)

             A       :   List[str]					=						AlbertTokenizer.from_pretrained(SCREAMING_SNAKE_CASE	)
         finally:
             os.remove(SCREAMING_SNAKE_CASE	)

         # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
         # the current folder and have the right name.
         if os.path.isfile('''tokenizer.json'''	):
             # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
             return
         try:
             with open('''tokenizer.json'''						,			'''wb'''	) as f:
                 http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''						,			SCREAMING_SNAKE_CASE	)
             A       :   List[Any]					=						AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''	)
             # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
             self.assertEqual(tokenizer.vocab_size						,			1000	)
             # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

         finally:
             os.remove('''tokenizer.json'''	)





     def 				__lowerCAmelCase							(							self	) ->				Any:






         """simple docstring"""






         A       :   int					=						AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''	)





@is_staging_test
class  A				(							unittest.TestCase						):
     __magic_name__						=		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']





     @classmethod
     def 				__lowerCAmelCase							(							cls	) ->				Optional[int]:






         """simple docstring"""






         A       :   List[str]					=						TOKEN
         HfFolder.save_token(SCREAMING_SNAKE_CASE	)





     @classmethod
     def 				__lowerCAmelCase							(							cls	) ->				List[str]:






         """simple docstring"""






         try:
             delete_repo(token=cls._token						,			repo_id='''test-tokenizer'''	)
         except HTTPError:
             pass

         try:
             delete_repo(token=cls._token						,			repo_id='''valid_org/test-tokenizer-org'''	)
         except HTTPError:
             pass

         try:
             delete_repo(token=cls._token						,			repo_id='''test-dynamic-tokenizer'''	)
         except HTTPError:
             pass





     def 				__lowerCAmelCase							(							self	) ->				List[str]:






         """simple docstring"""






         with tempfile.TemporaryDirectory() as tmp_dir:
             A       :   str					=						os.path.join(SCREAMING_SNAKE_CASE						,			'''vocab.txt'''	)
             with open(SCREAMING_SNAKE_CASE						,			'''w'''						,			encoding='''utf-8'''	) as vocab_writer:
                 vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]	)	)
             A       :   List[Any]					=						BertTokenizer(SCREAMING_SNAKE_CASE	)

         tokenizer.push_to_hub('''test-tokenizer'''						,			use_auth_token=self._token	)
         A       :   Optional[Any]					=						BertTokenizer.from_pretrained(F'{USER}/test-tokenizer'	)
         self.assertDictEqual(new_tokenizer.vocab						,			tokenizer.vocab	)

         # Reset repo
         delete_repo(token=self._token						,			repo_id='''test-tokenizer'''	)

         # Push to hub via save_pretrained
         with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer.save_pretrained(SCREAMING_SNAKE_CASE						,			repo_id='''test-tokenizer'''						,			push_to_hub=SCREAMING_SNAKE_CASE						,			use_auth_token=self._token	)

         A       :   int					=						BertTokenizer.from_pretrained(F'{USER}/test-tokenizer'	)
         self.assertDictEqual(new_tokenizer.vocab						,			tokenizer.vocab	)





     def 				__lowerCAmelCase							(							self	) ->				Any:






         """simple docstring"""






         with tempfile.TemporaryDirectory() as tmp_dir:
             A       :   Union[str, Any]					=						os.path.join(SCREAMING_SNAKE_CASE						,			'''vocab.txt'''	)
             with open(SCREAMING_SNAKE_CASE						,			'''w'''						,			encoding='''utf-8'''	) as vocab_writer:
                 vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]	)	)
             A       :   Optional[int]					=						BertTokenizer(SCREAMING_SNAKE_CASE	)

         tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''						,			use_auth_token=self._token	)
         A       :   List[Any]					=						BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''	)
         self.assertDictEqual(new_tokenizer.vocab						,			tokenizer.vocab	)

         # Reset repo
         delete_repo(token=self._token						,			repo_id='''valid_org/test-tokenizer-org'''	)

         # Push to hub via save_pretrained
         with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer.save_pretrained(
                 SCREAMING_SNAKE_CASE						,			repo_id='''valid_org/test-tokenizer-org'''						,			push_to_hub=SCREAMING_SNAKE_CASE						,			use_auth_token=self._token	)

         A       :   str					=						BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''	)
         self.assertDictEqual(new_tokenizer.vocab						,			tokenizer.vocab	)





     @require_tokenizers
     def 				__lowerCAmelCase							(							self	) ->				str:






         """simple docstring"""






         CustomTokenizer.register_for_auto_class()
         with tempfile.TemporaryDirectory() as tmp_dir:
             A       :   List[Any]					=						os.path.join(SCREAMING_SNAKE_CASE						,			'''vocab.txt'''	)
             with open(SCREAMING_SNAKE_CASE						,			'''w'''						,			encoding='''utf-8'''	) as vocab_writer:
                 vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]	)	)
             A       :   Optional[int]					=						CustomTokenizer(SCREAMING_SNAKE_CASE	)

         # No fast custom tokenizer
         tokenizer.push_to_hub('''test-dynamic-tokenizer'''						,			use_auth_token=self._token	)

         A       :   Dict					=						AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'						,			trust_remote_code=SCREAMING_SNAKE_CASE	)
         # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
         self.assertEqual(tokenizer.__class__.__name__						,			'''CustomTokenizer'''	)

         # Fast and slow custom tokenizer
         CustomTokenizerFast.register_for_auto_class()
         with tempfile.TemporaryDirectory() as tmp_dir:
             A       :   Optional[Any]					=						os.path.join(SCREAMING_SNAKE_CASE						,			'''vocab.txt'''	)
             with open(SCREAMING_SNAKE_CASE						,			'''w'''						,			encoding='''utf-8'''	) as vocab_writer:
                 vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]	)	)

             A       :   str					=						BertTokenizerFast.from_pretrained(SCREAMING_SNAKE_CASE	)
             bert_tokenizer.save_pretrained(SCREAMING_SNAKE_CASE	)
             A       :   Any					=						CustomTokenizerFast.from_pretrained(SCREAMING_SNAKE_CASE	)

         tokenizer.push_to_hub('''test-dynamic-tokenizer'''						,			use_auth_token=self._token	)

         A       :   Union[str, Any]					=						AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'						,			trust_remote_code=SCREAMING_SNAKE_CASE	)
         # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
         self.assertEqual(tokenizer.__class__.__name__						,			'''CustomTokenizerFast'''	)
         A       :   str					=						AutoTokenizer.from_pretrained(
             F'{USER}/test-dynamic-tokenizer'						,			use_fast=SCREAMING_SNAKE_CASE						,			trust_remote_code=SCREAMING_SNAKE_CASE	)
         # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
         self.assertEqual(tokenizer.__class__.__name__						,			'''CustomTokenizer'''	)





class  A				(							unittest.TestCase						):





     def 				__lowerCAmelCase							(							self	) ->				Dict:






         """simple docstring"""






         A       :   Optional[Any]					=						Trie()
         trie.add('''Hello 友達'''	)
         self.assertEqual(trie.data						,			{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}	)
         trie.add('''Hello'''	)
         trie.data
         self.assertEqual(trie.data						,			{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}	)





     def 				__lowerCAmelCase							(							self	) ->				List[str]:






         """simple docstring"""






         A       :   List[str]					=						Trie()
         self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''	)						,			['''[CLS] This is a extra_id_100''']	)
         trie.add('''[CLS]'''	)
         trie.add('''extra_id_1'''	)
         trie.add('''extra_id_100'''	)
         self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''	)						,			['''[CLS]''', ''' This is a ''', '''extra_id_100''']	)





     def 				__lowerCAmelCase							(							self	) ->				Tuple:






         """simple docstring"""






         A       :   Dict					=						Trie()
         trie.add('''A'''	)
         self.assertEqual(trie.split('''ABC'''	)						,			['''A''', '''BC''']	)
         self.assertEqual(trie.split('''BCA'''	)						,			['''BC''', '''A''']	)





     def 				__lowerCAmelCase							(							self	) ->				Dict:






         """simple docstring"""






         A       :   List[Any]					=						Trie()
         trie.add('''TOKEN]'''	)
         trie.add('''[SPECIAL_TOKEN]'''	)
         self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''	)						,			['''This is something ''', '''[SPECIAL_TOKEN]''']	)





     def 				__lowerCAmelCase							(							self	) ->				List[str]:






         """simple docstring"""






         A       :   str					=						Trie()
         trie.add('''A'''	)
         trie.add('''P'''	)
         trie.add('''[SPECIAL_TOKEN]'''	)
         self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''	)						,			['''This is something ''', '''[SPECIAL_TOKEN]''']	)





     def 				__lowerCAmelCase							(							self	) ->				Union[str, Any]:






         """simple docstring"""






         A       :   Any					=						Trie()
         trie.add('''AB'''	)
         trie.add('''B'''	)
         trie.add('''C'''	)
         self.assertEqual(trie.split('''ABC'''	)						,			['''AB''', '''C''']	)





     def 				__lowerCAmelCase							(							self	) ->				Any:






         """simple docstring"""






         A       :   List[str]					=						Trie()
         trie.add('''ABC'''	)
         trie.add('''B'''	)
         trie.add('''CD'''	)
         self.assertEqual(trie.split('''ABCD'''	)						,			['''ABC''', '''D''']	)





     def 				__lowerCAmelCase							(							self	) ->				Optional[int]:






         """simple docstring"""






         A       :   Tuple					=						Trie()
         A       :   Optional[int]					=						trie.cut_text('''ABC'''						,			[0, 0, 2, 1, 2, 3]	)
         self.assertEqual(SCREAMING_SNAKE_CASE						,			['''AB''', '''C''']	)









'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCamelCase__:   str								=			{
    "configuration_lxmert": ["LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "LxmertConfig"],
    "tokenization_lxmert": ["LxmertTokenizer"],
}

try:
						if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__:   int								=			["LxmertTokenizerFast"]

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__:   Union[str, Any]								=			[
						    "LxmertEncoder",
						    "LxmertForPreTraining",
						    "LxmertForQuestionAnswering",
						    "LxmertModel",
						    "LxmertPreTrainedModel",
						    "LxmertVisualFeatureEncoder",
						    "LxmertXLayer",
						]

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__:   int								=			[
						    "TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST",
						    "TFLxmertForPreTraining",
						    "TFLxmertMainLayer",
						    "TFLxmertModel",
						    "TFLxmertPreTrainedModel",
						    "TFLxmertVisualFeatureEncoder",
						]


if TYPE_CHECKING:
						from .configuration_lxmert import LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP, LxmertConfig
						from .tokenization_lxmert import LxmertTokenizer

						try:
												if not is_tokenizers_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_lxmert_fast import LxmertTokenizerFast

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_lxmert import (
												    LxmertEncoder,
												    LxmertForPreTraining,
												    LxmertForQuestionAnswering,
												    LxmertModel,
												    LxmertPreTrainedModel,
												    LxmertVisualFeatureEncoder,
												    LxmertXLayer,
												)

						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_tf_lxmert import (
												    TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    TFLxmertForPreTraining,
												    TFLxmertMainLayer,
												    TFLxmertModel,
												    TFLxmertPreTrainedModel,
												    TFLxmertVisualFeatureEncoder,
												)

else:
						import sys

						UpperCamelCase__:   Tuple								=			_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





