






from importlib import import_module

from .logging import get_logger


UpperCAmelCase__				   =		get_logger(__name__)






class 		lowercase_					:






      '''simple docstring'''




      def __init__(      self      :						int ,					__UpperCAmelCase      :						Optional[int] ,					__UpperCAmelCase      :						str=None  )    ->int:




       """simple docstring"""


       a		  =      attrs or []
       if module is not None:
        for key in module.__dict__:
         if key in attrs or not key.startswith('''__'''  ):
          setattr(self ,					__UpperCAmelCase ,					getattr(__UpperCAmelCase ,					__UpperCAmelCase  )  )
       a		  =      module._original_module if isinstance(__UpperCAmelCase ,					_PatchedModuleObj  ) else module






class 		lowercase_					:






      '''simple docstring'''




      __snake_case       =  []







      def __init__(      self      :						List[str] ,					__UpperCAmelCase      :						List[Any] ,					__UpperCAmelCase      :						str ,					__UpperCAmelCase      :						Optional[Any] ,					__UpperCAmelCase      :						Dict=None  )    ->Union[str, Any]:




       """simple docstring"""


       a		  =      obj
       a		  =      target
       a		  =      new
       a		  =      target.split('''.'''  )[0]
       a		  =      {}
       a		  =      attrs or []







      def __enter__(      self      :						Tuple  )    ->Union[str, Any]:




       """simple docstring"""


       *a  ,     a		  =      self.target.split('''.'''  )

       # Patch modules:
       # it's used to patch attributes of submodules like "os.path.join";
       # in this case we need to patch "os" and "os.path"

       for i in range(len(__UpperCAmelCase  )  ):
        try:
         a		  =      import_module('''.'''.join(submodules[: i + 1]  )  )
        except ModuleNotFoundError:
         continue
        # We iterate over all the globals in self.obj in case we find "os" or "os.path"
        for attr in self.obj.__dir__():
         a		  =      getattr(self.obj ,					__UpperCAmelCase  )
         # We don't check for the name of the global, but rather if its value *is* "os" or "os.path".
         # This allows to patch renamed modules like "from os import path as ospath".
         if obj_attr is submodule or (
             (isinstance(__UpperCAmelCase ,					_PatchedModuleObj  ) and obj_attr._original_module is submodule)
         ):
          a		  =      obj_attr
          # patch at top level
          setattr(self.obj ,					__UpperCAmelCase ,					_PatchedModuleObj(__UpperCAmelCase ,					attrs=self.attrs  )  )
          a		  =      getattr(self.obj ,					__UpperCAmelCase  )
          # construct lower levels patches
          for key in submodules[i + 1 :]:
           setattr(__UpperCAmelCase ,					__UpperCAmelCase ,					_PatchedModuleObj(getattr(__UpperCAmelCase ,					__UpperCAmelCase ,					__UpperCAmelCase  ) ,					attrs=self.attrs  )  )
           a		  =      getattr(__UpperCAmelCase ,					__UpperCAmelCase  )
          # finally set the target attribute
          setattr(__UpperCAmelCase ,					__UpperCAmelCase ,					self.new  )

        # Patch attribute itself:
        # it's used for builtins like "open",
        # and also to patch "os.path.join" we may also need to patch "join"
        # itself if it was imported as "from os.path import join".

       if submodules:  # if it's an attribute of a submodule like "os.path.join"
        try:
         a		  =      getattr(import_module('''.'''.join(__UpperCAmelCase  )  ) ,					__UpperCAmelCase  )
        except (AttributeError, ModuleNotFoundError):
         return
        # We iterate over all the globals in self.obj in case we find "os.path.join"
        for attr in self.obj.__dir__():
         # We don't check for the name of the global, but rather if its value *is* "os.path.join".
         # This allows to patch renamed attributes like "from os.path import join as pjoin".
         if getattr(self.obj ,					__UpperCAmelCase  ) is attr_value:
          a		  =      getattr(self.obj ,					__UpperCAmelCase  )
          setattr(self.obj ,					__UpperCAmelCase ,					self.new  )
       elif target_attr in globals()["__builtins__"]:  # if it'a s builtin like "open"
        a		  =      globals()['''__builtins__'''][target_attr]
        setattr(self.obj ,					__UpperCAmelCase ,					self.new  )
       else:
        raise RuntimeError(F"""Tried to patch attribute {target_attr} instead of a submodule."""  )







      def __exit__(      self      :						List[str] ,					*__UpperCAmelCase      :						Optional[int]  )    ->Tuple:




       """simple docstring"""


       for attr in list(self.original  ):
        setattr(self.obj ,					__UpperCAmelCase ,					self.original.pop(__UpperCAmelCase  )  )







      def       __lowerCAmelCase					(      self      :						Any  )    ->List[Any]:




       """simple docstring"""


       self.__enter__()
       self._active_patches.append(self  )







      def       __lowerCAmelCase					(      self      :						Any  )    ->Union[str, Any]:




       """simple docstring"""


       try:
        self._active_patches.remove(self  )
       except ValueError:
        # If the patch hasn't been started this will fail
        return None

       return self.__exit__()








import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_fnet import FNetTokenizer
else:
  _snake_case										=       None

_snake_case										=       logging.get_logger(__name__)
_snake_case										=       {"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

_snake_case										=       {
    "vocab_file": {
        "google/fnet-base": "https://huggingface.co/google/fnet-base/resolve/main/spiece.model",
        "google/fnet-large": "https://huggingface.co/google/fnet-large/resolve/main/spiece.model",
    },
    "tokenizer_file": {
        "google/fnet-base": "https://huggingface.co/google/fnet-base/resolve/main/tokenizer.json",
        "google/fnet-large": "https://huggingface.co/google/fnet-large/resolve/main/tokenizer.json",
    },
}

_snake_case										=       {
    "google/fnet-base": 512,
    "google/fnet-large": 512,
}

_snake_case										=       "▁"







class 						lowercase							(   UpperCamelCase__		):
     _a							 =						VOCAB_FILES_NAMES
     _a							 =						PRETRAINED_VOCAB_FILES_MAP
     _a							 =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _a							 =						["input_ids", "token_type_ids"]
     _a							 =						FNetTokenizer

     def __init__(  self		,     _a=None		,     _a=None		,     _a=False		,     _a=True		,     _a=True		,     _a="<unk>"		,     _a="[SEP]"		,     _a="<pad>"		,     _a="[CLS]"		,     _a="[MASK]"		,     **_a		,     )						->							Optional[int]:
           # Mask token behave like a normal word, i.e. include the space before it and
           # is included in the raw text, there should be a match in a non-normalized sentence.
           _A							:     int			=						(
               AddedToken(_a		,     lstrip=_a		,     rstrip=_a		,     normalized=_a				)
               if isinstance(_a		,     _a				)
               else mask_token
           )

           super().__init__(
               _a		,     tokenizer_file=_a		,     do_lower_case=_a		,     remove_space=_a		,     keep_accents=_a		,     unk_token=_a		,     sep_token=_a		,     pad_token=_a		,     cls_token=_a		,     mask_token=_a		,     **_a		,     )

           _A							:     Optional[int]			=						do_lower_case
           _A							:     List[Any]			=						remove_space
           _A							:     str			=						keep_accents
           _A							:     int			=						vocab_file
           _A							:     int			=						False if not self.vocab_file else True

     def 					a__	(  self		,     _a		,     _a = None				)						->							List[int]:
           _A							:     str			=						[self.sep_token_id]
           _A							:     Dict			=						[self.cls_token_id]
           if token_ids_a is None:
                 return cls + token_ids_a + sep
           return cls + token_ids_a + sep + token_ids_a + sep

     def 					a__	(  self		,     _a		,     _a = None				)						->							List[int]:
           _A							:     Any			=						[self.sep_token_id]
           _A							:     Any			=						[self.cls_token_id]

           if token_ids_a is None:
                 return len(cls + token_ids_a + sep				) * [0]
           return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]



     def 					a__	(  self		,     _a		,     _a = None				)						->							Tuple[str]:
           if not os.path.isdir(_a				):
                 logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''				)
                 return
           _A							:     List[str]			=						os.path.join(
               _a		,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)

           if os.path.abspath(self.vocab_file				) != os.path.abspath(_a				):
                 copyfile(self.vocab_file		,     _a				)

           return (out_vocab_file,)

