
"""simple docstring"""





import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
    from test_module.custom_tokenization_fast import CustomTokenizerFast


class A_	(unittest.TestCase      ):




  '''simple docstring'''







  def        UpperCamelCase__					(		self  ):

     """simple docstring"""
     # A mock response for an HTTP head request to emulate server down
     UpperCAmelCase_					:					Optional[int]			     =    mock.Mock()
     UpperCAmelCase_					:					Optional[Any]			     =    500
     UpperCAmelCase_					:					Dict			     =    {}
     UpperCAmelCase_					:					Union[str, Any]			     =    HTTPError
     UpperCAmelCase_					:					List[Any]			     =    {}

     # Download this model to make sure it's in the cache.
     UpperCAmelCase_					:					int			     =    BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"  )

     # Under the mock environment we get a 500 error when trying to reach the tokenizer.
     with mock.patch("requests.Session.request"				,   return_value=_a  ) as mock_head:
        UpperCAmelCase_					:					List[Any]			     =    BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"  )
        # This check we did call the fake head request
        mock_head.assert_called()







  @require_tokenizers
  def        UpperCamelCase__					(		self  ):

     """simple docstring"""
     # A mock response for an HTTP head request to emulate server down
     UpperCAmelCase_					:					str			     =    mock.Mock()
     UpperCAmelCase_					:					Any			     =    500
     UpperCAmelCase_					:					Optional[int]			     =    {}
     UpperCAmelCase_					:					List[str]			     =    HTTPError
     UpperCAmelCase_					:					int			     =    {}

     # Download this model to make sure it's in the cache.
     UpperCAmelCase_					:					int			     =    GPTaTokenizerFast.from_pretrained("gpt2"  )

     # Under the mock environment we get a 500 error when trying to reach the tokenizer.
     with mock.patch("requests.Session.request"				,   return_value=_a  ) as mock_head:
        UpperCAmelCase_					:					str			     =    GPTaTokenizerFast.from_pretrained("gpt2"  )
        # This check we did call the fake head request
        mock_head.assert_called()







  def        UpperCamelCase__					(		self  ):

     """simple docstring"""
     # This test is for deprecated behavior and can be removed in v5
     try:
        UpperCAmelCase_					:					Tuple			     =    tempfile.mktemp()
        with open(_a				,   "wb"  ) as f:
           http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"				,   _a  )

        UpperCAmelCase_					:					str			     =    AlbertTokenizer.from_pretrained(_a  )
     finally:
        os.remove(_a  )

     # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
     # the current folder and have the right name.
     if os.path.isfile("tokenizer.json"  ):
        # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
        return
     try:
        with open("tokenizer.json"				,   "wb"  ) as f:
           http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"				,   _a  )
        UpperCAmelCase_					:					Optional[int]			     =    AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"  )
        # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
        self.assertEqual(tokenizer.vocab_size				,   1000  )
        # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

     finally:
        os.remove("tokenizer.json"  )







  def        UpperCamelCase__					(		self  ):

     """simple docstring"""
     # This test is for deprecated behavior and can be removed in v5
     UpperCAmelCase_					:					Dict			     =    AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"  )




@is_staging_test
class A_	(unittest.TestCase      ):




  '''simple docstring'''
  SCREAMING_SNAKE_CASE__			:						List[Any]       =		["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]







  @classmethod
  def        UpperCamelCase__					(		cls  ):

     """simple docstring"""
     UpperCAmelCase_					:					int			     =    TOKEN
     HfFolder.save_token(_a  )







  @classmethod
  def        UpperCamelCase__					(		cls  ):

     """simple docstring"""
     try:
        delete_repo(token=cls._token				,   repo_id="test-tokenizer"  )
     except HTTPError:
        pass

     try:
        delete_repo(token=cls._token				,   repo_id="valid_org/test-tokenizer-org"  )
     except HTTPError:
        pass

     try:
        delete_repo(token=cls._token				,   repo_id="test-dynamic-tokenizer"  )
     except HTTPError:
        pass







  def        UpperCamelCase__					(		self  ):

     """simple docstring"""
     with tempfile.TemporaryDirectory() as tmp_dir:
        UpperCAmelCase_					:					Optional[Any]			     =    os.path.join(_a				,   "vocab.txt"  )
        with open(_a				,   "w"				,   encoding="utf-8"  ) as vocab_writer:
           vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]  )  )
        UpperCAmelCase_					:					Optional[Any]			     =    BertTokenizer(_a  )

     tokenizer.push_to_hub("test-tokenizer"				,   use_auth_token=self._token  )
     UpperCAmelCase_					:					List[Any]			     =    BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""  )
     self.assertDictEqual(new_tokenizer.vocab				,   tokenizer.vocab  )

     # Reset repo
     delete_repo(token=self._token				,   repo_id="test-tokenizer"  )

     # Push to hub via save_pretrained
     with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(_a				,   repo_id="test-tokenizer"				,   push_to_hub=_a				,   use_auth_token=self._token  )

     UpperCAmelCase_					:					List[str]			     =    BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""  )
     self.assertDictEqual(new_tokenizer.vocab				,   tokenizer.vocab  )







  def        UpperCamelCase__					(		self  ):

     """simple docstring"""
     with tempfile.TemporaryDirectory() as tmp_dir:
        UpperCAmelCase_					:					Optional[int]			     =    os.path.join(_a				,   "vocab.txt"  )
        with open(_a				,   "w"				,   encoding="utf-8"  ) as vocab_writer:
           vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]  )  )
        UpperCAmelCase_					:					Tuple			     =    BertTokenizer(_a  )

     tokenizer.push_to_hub("valid_org/test-tokenizer-org"				,   use_auth_token=self._token  )
     UpperCAmelCase_					:					List[Any]			     =    BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"  )
     self.assertDictEqual(new_tokenizer.vocab				,   tokenizer.vocab  )

     # Reset repo
     delete_repo(token=self._token				,   repo_id="valid_org/test-tokenizer-org"  )

     # Push to hub via save_pretrained
     with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(
            _a				,   repo_id="valid_org/test-tokenizer-org"				,   push_to_hub=_a				,   use_auth_token=self._token  )

     UpperCAmelCase_					:					Dict			     =    BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"  )
     self.assertDictEqual(new_tokenizer.vocab				,   tokenizer.vocab  )







  @require_tokenizers
  def        UpperCamelCase__					(		self  ):

     """simple docstring"""
     CustomTokenizer.register_for_auto_class()
     with tempfile.TemporaryDirectory() as tmp_dir:
        UpperCAmelCase_					:					Union[str, Any]			     =    os.path.join(_a				,   "vocab.txt"  )
        with open(_a				,   "w"				,   encoding="utf-8"  ) as vocab_writer:
           vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]  )  )
        UpperCAmelCase_					:					str			     =    CustomTokenizer(_a  )

     # No fast custom tokenizer
     tokenizer.push_to_hub("test-dynamic-tokenizer"				,   use_auth_token=self._token  )

     UpperCAmelCase_					:					Optional[int]			     =    AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""				,   trust_remote_code=_a  )
     # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
     self.assertEqual(tokenizer.__class__.__name__				,   "CustomTokenizer"  )

     # Fast and slow custom tokenizer
     CustomTokenizerFast.register_for_auto_class()
     with tempfile.TemporaryDirectory() as tmp_dir:
        UpperCAmelCase_					:					Any			     =    os.path.join(_a				,   "vocab.txt"  )
        with open(_a				,   "w"				,   encoding="utf-8"  ) as vocab_writer:
           vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]  )  )

        UpperCAmelCase_					:					Dict			     =    BertTokenizerFast.from_pretrained(_a  )
        bert_tokenizer.save_pretrained(_a  )
        UpperCAmelCase_					:					Dict			     =    CustomTokenizerFast.from_pretrained(_a  )

     tokenizer.push_to_hub("test-dynamic-tokenizer"				,   use_auth_token=self._token  )

     UpperCAmelCase_					:					Tuple			     =    AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""				,   trust_remote_code=_a  )
     # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
     self.assertEqual(tokenizer.__class__.__name__				,   "CustomTokenizerFast"  )
     UpperCAmelCase_					:					Tuple			     =    AutoTokenizer.from_pretrained(
         F"""{USER}/test-dynamic-tokenizer"""				,   use_fast=_a				,   trust_remote_code=_a  )
     # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
     self.assertEqual(tokenizer.__class__.__name__				,   "CustomTokenizer"  )




class A_	(unittest.TestCase      ):




  '''simple docstring'''







  def        UpperCamelCase__					(		self  ):

     """simple docstring"""
     UpperCAmelCase_					:					Optional[Any]			     =    Trie()
     trie.add("Hello 友達"  )
     self.assertEqual(trie.data				,   {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}  )
     trie.add("Hello"  )
     trie.data
     self.assertEqual(trie.data				,   {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}  )







  def        UpperCamelCase__					(		self  ):

     """simple docstring"""
     UpperCAmelCase_					:					List[Any]			     =    Trie()
     self.assertEqual(trie.split("[CLS] This is a extra_id_100"  )				,   ["[CLS] This is a extra_id_100"]  )
     trie.add("[CLS]"  )
     trie.add("extra_id_1"  )
     trie.add("extra_id_100"  )
     self.assertEqual(trie.split("[CLS] This is a extra_id_100"  )				,   ["[CLS]", " This is a ", "extra_id_100"]  )







  def        UpperCamelCase__					(		self  ):

     """simple docstring"""
     UpperCAmelCase_					:					List[Any]			     =    Trie()
     trie.add("A"  )
     self.assertEqual(trie.split("ABC"  )				,   ["A", "BC"]  )
     self.assertEqual(trie.split("BCA"  )				,   ["BC", "A"]  )







  def        UpperCamelCase__					(		self  ):

     """simple docstring"""
     UpperCAmelCase_					:					Dict			     =    Trie()
     trie.add("TOKEN]"  )
     trie.add("[SPECIAL_TOKEN]"  )
     self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"  )				,   ["This is something ", "[SPECIAL_TOKEN]"]  )







  def        UpperCamelCase__					(		self  ):

     """simple docstring"""
     UpperCAmelCase_					:					Optional[Any]			     =    Trie()
     trie.add("A"  )
     trie.add("P"  )
     trie.add("[SPECIAL_TOKEN]"  )
     self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"  )				,   ["This is something ", "[SPECIAL_TOKEN]"]  )







  def        UpperCamelCase__					(		self  ):

     """simple docstring"""
     UpperCAmelCase_					:					Dict			     =    Trie()
     trie.add("AB"  )
     trie.add("B"  )
     trie.add("C"  )
     self.assertEqual(trie.split("ABC"  )				,   ["AB", "C"]  )







  def        UpperCamelCase__					(		self  ):

     """simple docstring"""
     UpperCAmelCase_					:					Any			     =    Trie()
     trie.add("ABC"  )
     trie.add("B"  )
     trie.add("CD"  )
     self.assertEqual(trie.split("ABCD"  )				,   ["ABC", "D"]  )







  def        UpperCamelCase__					(		self  ):

     """simple docstring"""
     # Even if the offsets are wrong, we necessarily output correct string
     # parts.
     UpperCAmelCase_					:					Tuple			     =    Trie()
     UpperCAmelCase_					:					int			     =    trie.cut_text("ABC"				,   [0, 0, 2, 1, 2, 3]  )
     self.assertEqual(_a				,   ["AB", "C"]  )








from ...configuration_utils import PretrainedConfig
from ...utils import logging


_snake_case										=       logging.get_logger(__name__)

_snake_case										=       {
    "google/fnet-base": "https://huggingface.co/google/fnet-base/resolve/main/config.json",
    "google/fnet-large": "https://huggingface.co/google/fnet-large/resolve/main/config.json"
    # See all FNet models at https://huggingface.co/models?filter=fnet
}







class 						lowercase							(   UpperCamelCase__		):
     _a							 =						"fnet"

     def __init__(  self		,     _a=3_2000		,     _a=768		,     _a=12		,     _a=3072		,     _a="gelu_new"		,     _a=0.1		,     _a=512		,     _a=4		,     _a=0.02		,     _a=1e-12		,     _a=False		,     _a=512		,     _a=3		,     _a=1		,     _a=2		,     **_a		,     )						->							int:
           super().__init__(pad_token_id=_a		,     bos_token_id=_a		,     eos_token_id=_a		,     **_a				)

           _A							:     Any			=						vocab_size
           _A							:     str			=						max_position_embeddings
           _A							:     Optional[Any]			=						hidden_size
           _A							:     List[str]			=						num_hidden_layers
           _A							:     List[str]			=						intermediate_size
           _A							:     List[Any]			=						hidden_act
           _A							:     List[str]			=						hidden_dropout_prob
           _A							:     List[str]			=						initializer_range
           _A							:     List[Any]			=						type_vocab_size
           _A							:     List[Any]			=						layer_norm_eps
           _A							:     List[str]			=						use_tpu_fourier_optimizations
           _A							:     str			=						tpu_short_seq_length

