'''simple docstring'''






import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
     from test_module.custom_tokenization_fast import CustomTokenizerFast





class     A__							(       unittest.TestCase   ):





   """simple docstring"""





   def 			_lowerCAmelCase		( self     :		str			)       ->    Optional[int]:







      """simple docstring"""




      _UpperCAmelCase							:	Any							=   mock.Mock()
      _UpperCAmelCase							:	int							=   5_0_0
      _UpperCAmelCase							:	Optional[Any]							=   {}
      _UpperCAmelCase							:	List[str]							=   HTTPError
      _UpperCAmelCase							:	Tuple							=   {}

      # Download this model to make sure it's in the cache.
      _UpperCAmelCase							:	Tuple							=   BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"			)

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch("requests.Session.request"						,     return_value=_a			) as mock_head:
         _UpperCAmelCase							:	List[Any]							=   BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"			)
         # This check we did call the fake head request
         mock_head.assert_called()




   @require_tokenizers
   def 			_lowerCAmelCase		( self     :		int			)       ->    Tuple:







      """simple docstring"""




      _UpperCAmelCase							:	str							=   mock.Mock()
      _UpperCAmelCase							:	Union[str, Any]							=   5_0_0
      _UpperCAmelCase							:	Dict							=   {}
      _UpperCAmelCase							:	List[str]							=   HTTPError
      _UpperCAmelCase							:	int							=   {}

      # Download this model to make sure it's in the cache.
      _UpperCAmelCase							:	str							=   GPTaTokenizerFast.from_pretrained("gpt2"			)

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch("requests.Session.request"						,     return_value=_a			) as mock_head:
         _UpperCAmelCase							:	int							=   GPTaTokenizerFast.from_pretrained("gpt2"			)
         # This check we did call the fake head request
         mock_head.assert_called()




   def 			_lowerCAmelCase		( self     :		Tuple			)       ->    str:







      """simple docstring"""




      try:
         _UpperCAmelCase							:	Optional[Any]							=   tempfile.mktemp()
         with open(_a						,     "wb"			) as f:
            http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"						,     _a			)

         _UpperCAmelCase							:	int							=   AlbertTokenizer.from_pretrained(_a			)
      finally:
         os.remove(_a			)

      # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
      # the current folder and have the right name.
      if os.path.isfile("tokenizer.json"			):
         # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
         return
      try:
         with open("tokenizer.json"						,     "wb"			) as f:
            http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"						,     _a			)
         _UpperCAmelCase							:	Union[str, Any]							=   AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"			)
         # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
         self.assertEqual(tokenizer.vocab_size						,     1_0_0_0			)
         # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

      finally:
         os.remove("tokenizer.json"			)




   def 			_lowerCAmelCase		( self     :		Optional[int]			)       ->    int:







      """simple docstring"""




      _UpperCAmelCase							:	List[str]							=   AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"			)





@is_staging_test
class     A__							(       unittest.TestCase   ):





   """simple docstring"""





   UpperCamelCase_       :						List[str]    				=     ['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']




   @classmethod
   def 			_lowerCAmelCase		( cls     :		Any			)       ->    Optional[Any]:







      """simple docstring"""




      _UpperCAmelCase							:	int							=   TOKEN
      HfFolder.save_token(_a			)




   @classmethod
   def 			_lowerCAmelCase		( cls     :		Tuple			)       ->    Optional[int]:







      """simple docstring"""




      try:
         delete_repo(token=cls._token						,     repo_id="test-tokenizer"			)
      except HTTPError:
         pass

      try:
         delete_repo(token=cls._token						,     repo_id="valid_org/test-tokenizer-org"			)
      except HTTPError:
         pass

      try:
         delete_repo(token=cls._token						,     repo_id="test-dynamic-tokenizer"			)
      except HTTPError:
         pass




   def 			_lowerCAmelCase		( self     :		Any			)       ->    Optional[Any]:







      """simple docstring"""




      with tempfile.TemporaryDirectory() as tmp_dir:
         _UpperCAmelCase							:	List[Any]							=   os.path.join(_a						,     "vocab.txt"			)
         with open(_a						,     "w"						,     encoding="utf-8"			) as vocab_writer:
            vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]			)			)
         _UpperCAmelCase							:	int							=   BertTokenizer(_a			)

      tokenizer.push_to_hub("test-tokenizer"						,     use_auth_token=self._token			)
      _UpperCAmelCase							:	Dict							=   BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""			)
      self.assertDictEqual(new_tokenizer.vocab						,     tokenizer.vocab			)

      # Reset repo
      delete_repo(token=self._token						,     repo_id="test-tokenizer"			)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
         tokenizer.save_pretrained(_a						,     repo_id="test-tokenizer"						,     push_to_hub=_a						,     use_auth_token=self._token			)

      _UpperCAmelCase							:	Tuple							=   BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""			)
      self.assertDictEqual(new_tokenizer.vocab						,     tokenizer.vocab			)




   def 			_lowerCAmelCase		( self     :		Tuple			)       ->    Any:







      """simple docstring"""




      with tempfile.TemporaryDirectory() as tmp_dir:
         _UpperCAmelCase							:	Dict							=   os.path.join(_a						,     "vocab.txt"			)
         with open(_a						,     "w"						,     encoding="utf-8"			) as vocab_writer:
            vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]			)			)
         _UpperCAmelCase							:	Optional[Any]							=   BertTokenizer(_a			)

      tokenizer.push_to_hub("valid_org/test-tokenizer-org"						,     use_auth_token=self._token			)
      _UpperCAmelCase							:	int							=   BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"			)
      self.assertDictEqual(new_tokenizer.vocab						,     tokenizer.vocab			)

      # Reset repo
      delete_repo(token=self._token						,     repo_id="valid_org/test-tokenizer-org"			)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
         tokenizer.save_pretrained(
             _a						,     repo_id="valid_org/test-tokenizer-org"						,     push_to_hub=_a						,     use_auth_token=self._token			)

      _UpperCAmelCase							:	Any							=   BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"			)
      self.assertDictEqual(new_tokenizer.vocab						,     tokenizer.vocab			)




   @require_tokenizers
   def 			_lowerCAmelCase		( self     :		Optional[int]			)       ->    Optional[int]:







      """simple docstring"""




      CustomTokenizer.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
         _UpperCAmelCase							:	Tuple							=   os.path.join(_a						,     "vocab.txt"			)
         with open(_a						,     "w"						,     encoding="utf-8"			) as vocab_writer:
            vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]			)			)
         _UpperCAmelCase							:	List[Any]							=   CustomTokenizer(_a			)

      # No fast custom tokenizer
      tokenizer.push_to_hub("test-dynamic-tokenizer"						,     use_auth_token=self._token			)

      _UpperCAmelCase							:	Dict							=   AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""						,     trust_remote_code=_a			)
      # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__						,     "CustomTokenizer"			)

      # Fast and slow custom tokenizer
      CustomTokenizerFast.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
         _UpperCAmelCase							:	Dict							=   os.path.join(_a						,     "vocab.txt"			)
         with open(_a						,     "w"						,     encoding="utf-8"			) as vocab_writer:
            vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]			)			)

         _UpperCAmelCase							:	Dict							=   BertTokenizerFast.from_pretrained(_a			)
         bert_tokenizer.save_pretrained(_a			)
         _UpperCAmelCase							:	List[Any]							=   CustomTokenizerFast.from_pretrained(_a			)

      tokenizer.push_to_hub("test-dynamic-tokenizer"						,     use_auth_token=self._token			)

      _UpperCAmelCase							:	str							=   AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""						,     trust_remote_code=_a			)
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__						,     "CustomTokenizerFast"			)
      _UpperCAmelCase							:	Optional[int]							=   AutoTokenizer.from_pretrained(
          F"""{USER}/test-dynamic-tokenizer"""						,     use_fast=_a						,     trust_remote_code=_a			)
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__						,     "CustomTokenizer"			)





class     A__							(       unittest.TestCase   ):





   """simple docstring"""





   def 			_lowerCAmelCase		( self     :		Any			)       ->    int:







      """simple docstring"""




      _UpperCAmelCase							:	int							=   Trie()
      trie.add("Hello 友達"			)
      self.assertEqual(trie.data						,     {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}			)
      trie.add("Hello"			)
      trie.data
      self.assertEqual(trie.data						,     {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}			)




   def 			_lowerCAmelCase		( self     :		str			)       ->    Union[str, Any]:







      """simple docstring"""




      _UpperCAmelCase							:	Optional[int]							=   Trie()
      self.assertEqual(trie.split("[CLS] This is a extra_id_100"			)						,     ["[CLS] This is a extra_id_100"]			)
      trie.add("[CLS]"			)
      trie.add("extra_id_1"			)
      trie.add("extra_id_100"			)
      self.assertEqual(trie.split("[CLS] This is a extra_id_100"			)						,     ["[CLS]", " This is a ", "extra_id_100"]			)




   def 			_lowerCAmelCase		( self     :		Union[str, Any]			)       ->    str:







      """simple docstring"""




      _UpperCAmelCase							:	Optional[Any]							=   Trie()
      trie.add("A"			)
      self.assertEqual(trie.split("ABC"			)						,     ["A", "BC"]			)
      self.assertEqual(trie.split("BCA"			)						,     ["BC", "A"]			)




   def 			_lowerCAmelCase		( self     :		List[str]			)       ->    Any:







      """simple docstring"""




      _UpperCAmelCase							:	List[Any]							=   Trie()
      trie.add("TOKEN]"			)
      trie.add("[SPECIAL_TOKEN]"			)
      self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"			)						,     ["This is something ", "[SPECIAL_TOKEN]"]			)




   def 			_lowerCAmelCase		( self     :		Optional[Any]			)       ->    Any:







      """simple docstring"""




      _UpperCAmelCase							:	Optional[int]							=   Trie()
      trie.add("A"			)
      trie.add("P"			)
      trie.add("[SPECIAL_TOKEN]"			)
      self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"			)						,     ["This is something ", "[SPECIAL_TOKEN]"]			)




   def 			_lowerCAmelCase		( self     :		List[Any]			)       ->    List[str]:







      """simple docstring"""




      _UpperCAmelCase							:	Dict							=   Trie()
      trie.add("AB"			)
      trie.add("B"			)
      trie.add("C"			)
      self.assertEqual(trie.split("ABC"			)						,     ["AB", "C"]			)




   def 			_lowerCAmelCase		( self     :		Optional[Any]			)       ->    Union[str, Any]:







      """simple docstring"""




      _UpperCAmelCase							:	int							=   Trie()
      trie.add("ABC"			)
      trie.add("B"			)
      trie.add("CD"			)
      self.assertEqual(trie.split("ABCD"			)						,     ["ABC", "D"]			)




   def 			_lowerCAmelCase		( self     :		Optional[Any]			)       ->    Any:







      """simple docstring"""




      _UpperCAmelCase							:	Union[str, Any]							=   Trie()
      _UpperCAmelCase							:	Union[str, Any]							=   trie.cut_text("ABC"						,     [0, 0, 2, 1, 2, 3]			)
      self.assertEqual(_a						,     ["AB", "C"]			)



"""simple docstring"""




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowercase_			     =  {
    "configuration_roformer": ["ROFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP", "RoFormerConfig", "RoFormerOnnxConfig"],
    "tokenization_roformer": ["RoFormerTokenizer"],
}

try:
				if not is_tokenizers_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_			     =  ["RoFormerTokenizerFast"]

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_			     =  [
				    "ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST",
				    "RoFormerForCausalLM",
				    "RoFormerForMaskedLM",
				    "RoFormerForMultipleChoice",
				    "RoFormerForQuestionAnswering",
				    "RoFormerForSequenceClassification",
				    "RoFormerForTokenClassification",
				    "RoFormerLayer",
				    "RoFormerModel",
				    "RoFormerPreTrainedModel",
				    "load_tf_weights_in_roformer",
				]


try:
				if not is_tf_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_			     =  [
				    "TF_ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST",
				    "TFRoFormerForCausalLM",
				    "TFRoFormerForMaskedLM",
				    "TFRoFormerForMultipleChoice",
				    "TFRoFormerForQuestionAnswering",
				    "TFRoFormerForSequenceClassification",
				    "TFRoFormerForTokenClassification",
				    "TFRoFormerLayer",
				    "TFRoFormerModel",
				    "TFRoFormerPreTrainedModel",
				]


try:
				if not is_flax_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_			     =  [
				    "FLAX_ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST",
				    "FlaxRoFormerForMaskedLM",
				    "FlaxRoFormerForMultipleChoice",
				    "FlaxRoFormerForQuestionAnswering",
				    "FlaxRoFormerForSequenceClassification",
				    "FlaxRoFormerForTokenClassification",
				    "FlaxRoFormerModel",
				    "FlaxRoFormerPreTrainedModel",
				]


if TYPE_CHECKING:
				from .configuration_roformer import ROFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, RoFormerConfig, RoFormerOnnxConfig
				from .tokenization_roformer import RoFormerTokenizer

				try:
								if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_roformer_fast import RoFormerTokenizerFast

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_roformer import (
								    ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
								    RoFormerForCausalLM,
								    RoFormerForMaskedLM,
								    RoFormerForMultipleChoice,
								    RoFormerForQuestionAnswering,
								    RoFormerForSequenceClassification,
								    RoFormerForTokenClassification,
								    RoFormerLayer,
								    RoFormerModel,
								    RoFormerPreTrainedModel,
								    load_tf_weights_in_roformer,
								)

				try:
								if not is_tf_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_tf_roformer import (
								    TF_ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
								    TFRoFormerForCausalLM,
								    TFRoFormerForMaskedLM,
								    TFRoFormerForMultipleChoice,
								    TFRoFormerForQuestionAnswering,
								    TFRoFormerForSequenceClassification,
								    TFRoFormerForTokenClassification,
								    TFRoFormerLayer,
								    TFRoFormerModel,
								    TFRoFormerPreTrainedModel,
								)

				try:
								if not is_flax_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_flax_roformer import (
								    FLAX_ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
								    FlaxRoFormerForMaskedLM,
								    FlaxRoFormerForMultipleChoice,
								    FlaxRoFormerForQuestionAnswering,
								    FlaxRoFormerForSequenceClassification,
								    FlaxRoFormerForTokenClassification,
								    FlaxRoFormerModel,
								    FlaxRoFormerPreTrainedModel,
								)


else:
				import sys

				lowercase_			     =  _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)






