

import os
import re
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__							=			logging.get_logger(__name__)
lowerCAmelCase__							=			{"vocab_file": "spiece.model"}

lowerCAmelCase__							=			{
    "vocab_file": {
        "google/fnet-base": "https://huggingface.co/google/fnet-base/resolve/main/spiece.model",
        "google/fnet-large": "https://huggingface.co/google/fnet-large/resolve/main/spiece.model",
    },
}

lowerCAmelCase__							=			{
    "google/fnet-base": 512,
    "google/fnet-large": 512,
}

lowerCAmelCase__							=			"▁"






class     __magic_name__     (							_snake_case     ):
 UpperCAmelCase				     =    VOCAB_FILES_NAMES
 UpperCAmelCase				     =    PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase				     =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase				     =    ["""input_ids""", """token_type_ids"""]


 def __init__(    self		:			Union[str, Any]      ,    lowerCAmelCase__		:			List[Any]      ,    lowerCAmelCase__		:			Tuple=False      ,    lowerCAmelCase__		:			str=True      ,    lowerCAmelCase__		:			Optional[int]=True      ,    lowerCAmelCase__		:			str="<unk>"      ,    lowerCAmelCase__		:			Tuple="[SEP]"      ,    lowerCAmelCase__		:			Optional[Any]="<pad>"      ,    lowerCAmelCase__		:			int="[CLS]"      ,    lowerCAmelCase__		:			Tuple="[MASK]"      ,    lowerCAmelCase__		:			Optional[Dict[str, Any]] = None      ,    **lowerCAmelCase__		:			List[str]      ,    )     ->   None:
   # Mask token behave like a normal word, i.e. include the space before it and
   # is included in the raw text, there should be a match in a non-normalized sentence.
   UpperCAmelCase          =	(
       AddedToken(lowerCAmelCase__      ,    lstrip=lowerCAmelCase__      ,    rstrip=lowerCAmelCase__      ,    normalized=lowerCAmelCase__     )
       if isinstance(lowerCAmelCase__      ,    lowerCAmelCase__     )
       else mask_token
   )

   UpperCAmelCase          =	{} if sp_model_kwargs is None else sp_model_kwargs

   super().__init__(
       do_lower_case=lowerCAmelCase__      ,    remove_space=lowerCAmelCase__      ,    keep_accents=lowerCAmelCase__      ,    unk_token=lowerCAmelCase__      ,    sep_token=lowerCAmelCase__      ,    pad_token=lowerCAmelCase__      ,    cls_token=lowerCAmelCase__      ,    mask_token=lowerCAmelCase__      ,    sp_model_kwargs=self.sp_model_kwargs      ,    **lowerCAmelCase__      ,    )

   UpperCAmelCase          =	do_lower_case
   UpperCAmelCase          =	remove_space
   UpperCAmelCase          =	keep_accents
   UpperCAmelCase          =	vocab_file

   UpperCAmelCase          =	spm.SentencePieceProcessor(**self.sp_model_kwargs     )
   self.sp_model.Load(lowerCAmelCase__     )


 @property
 def _UpperCamelCase						(    self		:			List[Any]     )     ->   Optional[int]:
   return len(self.sp_model     )


 def _UpperCamelCase						(    self		:			str     )     ->   List[str]:
   UpperCAmelCase          =	{self.convert_ids_to_tokens(lowerCAmelCase__     ): i for i in range(self.vocab_size     )}
   vocab.update(self.added_tokens_encoder     )
   return vocab


 def __getstate__(    self		:			Dict     )     ->   Any:
   UpperCAmelCase          =	self.__dict__.copy()
   UpperCAmelCase          =	None
   return state


 def __setstate__(    self		:			Union[str, Any]      ,    lowerCAmelCase__		:			List[str]     )     ->   List[Any]:
   UpperCAmelCase          =	d

   # for backward compatibility
   if not hasattr(self      ,    "sp_model_kwargs"     ):
     UpperCAmelCase          =	{}

   UpperCAmelCase          =	spm.SentencePieceProcessor(**self.sp_model_kwargs     )
   self.sp_model.Load(self.vocab_file     )


 def _UpperCamelCase						(    self		:			int      ,    lowerCAmelCase__		:			Tuple     )     ->   List[str]:
   if self.remove_space:
     UpperCAmelCase          =	" ".join(inputs.strip().split()     )
   else:
     UpperCAmelCase          =	inputs
   UpperCAmelCase          =	outputs.replace("``"      ,    "\""     ).replace("''"      ,    "\""     )

   if not self.keep_accents:
     UpperCAmelCase          =	unicodedata.normalize("NFKD"      ,    lowerCAmelCase__     )
     UpperCAmelCase          =	"".join([c for c in outputs if not unicodedata.combining(lowerCAmelCase__     )]     )
   if self.do_lower_case:
     UpperCAmelCase          =	outputs.lower()

   return outputs


 def _UpperCamelCase						(    self		:			List[Any]      ,    lowerCAmelCase__		:			str     )     ->   List[str]:
   UpperCAmelCase          =	self.preprocess_text(lowerCAmelCase__     )
   UpperCAmelCase          =	self.sp_model.encode(lowerCAmelCase__      ,    out_type=lowerCAmelCase__     )
   UpperCAmelCase          =	[]
   for piece in pieces:
     if len(lowerCAmelCase__     ) > 1 and piece[-1] == str(","     ) and piece[-2].isdigit():
       UpperCAmelCase          =	self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCAmelCase__      ,    ""     )     )
       if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
         if len(cur_pieces[0]     ) == 1:
           UpperCAmelCase          =	cur_pieces[1:]
         else:
           UpperCAmelCase          =	cur_pieces[0][1:]
       cur_pieces.append(piece[-1]     )
       new_pieces.extend(lowerCAmelCase__     )
     else:
       new_pieces.append(lowerCAmelCase__     )

   return new_pieces


 def _UpperCamelCase						(    self		:			Any      ,    lowerCAmelCase__		:			Tuple     )     ->   Union[str, Any]:
   return self.sp_model.PieceToId(lowerCAmelCase__     )


 def _UpperCamelCase						(    self		:			Optional[Any]      ,    lowerCAmelCase__		:			Dict     )     ->   Tuple:
   return self.sp_model.IdToPiece(lowerCAmelCase__     )


 def _UpperCamelCase						(    self		:			Tuple      ,    lowerCAmelCase__		:			int     )     ->   Dict:
   UpperCAmelCase          =	[]
   UpperCAmelCase          =	""
   UpperCAmelCase          =	False
   for token in tokens:
     # make sure that special tokens are not decoded using sentencepiece model
     if token in self.all_special_tokens:
       if not prev_is_special:
         out_string += " "
       out_string += self.sp_model.decode(lowerCAmelCase__     ) + token
       UpperCAmelCase          =	True
       UpperCAmelCase          =	[]
     else:
       current_sub_tokens.append(lowerCAmelCase__     )
       UpperCAmelCase          =	False
   out_string += self.sp_model.decode(lowerCAmelCase__     )
   return out_string.strip()


 def _UpperCamelCase						(    self		:			Optional[int]      ,    lowerCAmelCase__		:			List[int]      ,    lowerCAmelCase__		:			bool = False      ,    lowerCAmelCase__		:			bool = None      ,    lowerCAmelCase__		:			bool = True      ,    **lowerCAmelCase__		:			Dict      ,    )     ->   str:
   UpperCAmelCase          =	kwargs.pop("use_source_tokenizer"      ,    lowerCAmelCase__     )

   UpperCAmelCase          =	self.convert_ids_to_tokens(lowerCAmelCase__      ,    skip_special_tokens=lowerCAmelCase__     )

   # To avoid mixing byte-level and unicode for byte-level BPT
   # we need to build string separately for added tokens and byte-level tokens
   # cf. https://github.com/huggingface/transformers/issues/1133
   UpperCAmelCase          =	[]
   UpperCAmelCase          =	[]
   for token in filtered_tokens:
     if skip_special_tokens and token in self.all_special_ids:
       continue
     if token in self.added_tokens_encoder:
       if current_sub_text:
         sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase__     )     )
         UpperCAmelCase          =	[]
       sub_texts.append(lowerCAmelCase__     )
     else:
       current_sub_text.append(lowerCAmelCase__     )
   if current_sub_text:
     sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase__     )     )

   # Mimic the behavior of the Rust tokenizer:
   # No space after <unk>
   if spaces_between_special_tokens:
     UpperCAmelCase          =	re.sub(r"(<unk>) "      ,    r"\1"      ,    " ".join(lowerCAmelCase__     )     )
   else:
     UpperCAmelCase          =	"".join(lowerCAmelCase__     )

   UpperCAmelCase          =	(
       clean_up_tokenization_spaces
       if clean_up_tokenization_spaces is not None
       else self.clean_up_tokenization_spaces
   )
   if clean_up_tokenization_spaces:
     UpperCAmelCase          =	self.clean_up_tokenization(lowerCAmelCase__     )
     return clean_text
   else:
     return text


 def _UpperCamelCase						(    self		:			Optional[Any]      ,    lowerCAmelCase__		:			List[int]      ,    lowerCAmelCase__		:			Optional[List[int]] = None     )     ->   List[int]:
   UpperCAmelCase          =	[self.sep_token_id]
   UpperCAmelCase          =	[self.cls_token_id]
   if token_ids_a is None:
     return cls + token_ids_a + sep
   return cls + token_ids_a + sep + token_ids_a + sep


 def _UpperCamelCase						(    self		:			Union[str, Any]      ,    lowerCAmelCase__		:			List[int]      ,    lowerCAmelCase__		:			Optional[List[int]] = None      ,    lowerCAmelCase__		:			bool = False     )     ->   List[int]:

   if already_has_special_tokens:
     return super().get_special_tokens_mask(
         token_ids_a=lowerCAmelCase__      ,    token_ids_a=lowerCAmelCase__      ,    already_has_special_tokens=lowerCAmelCase__     )

   if token_ids_a is not None:
     return [1] + ([0] * len(lowerCAmelCase__     )) + [1] + ([0] * len(lowerCAmelCase__     )) + [1]
   return [1] + ([0] * len(lowerCAmelCase__     )) + [1]


 def _UpperCamelCase						(    self		:			Optional[int]      ,    lowerCAmelCase__		:			List[int]      ,    lowerCAmelCase__		:			Optional[List[int]] = None     )     ->   List[int]:
   UpperCAmelCase          =	[self.sep_token_id]
   UpperCAmelCase          =	[self.cls_token_id]

   if token_ids_a is None:
     return len(cls + token_ids_a + sep     ) * [0]
   return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]


 def _UpperCamelCase						(    self		:			List[Any]      ,    lowerCAmelCase__		:			str      ,    lowerCAmelCase__		:			Optional[str] = None     )     ->   Tuple[str]:
   if not os.path.isdir(lowerCAmelCase__     ):
     logger.error(f"Vocabulary path ({save_directory}) should be a directory"     )
     return
   UpperCAmelCase          =	os.path.join(
       lowerCAmelCase__      ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )

   if os.path.abspath(self.vocab_file     ) != os.path.abspath(lowerCAmelCase__     ) and os.path.isfile(self.vocab_file     ):
     copyfile(self.vocab_file      ,    lowerCAmelCase__     )
   elif not os.path.isfile(self.vocab_file     ):
     with open(lowerCAmelCase__      ,    "wb"     ) as fi:
       UpperCAmelCase          =	self.sp_model.serialized_model_proto()
       fi.write(lowerCAmelCase__     )

   return (out_vocab_file,)






from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    convert_to_rgb,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


lowerCAmelCase__							=			logging.get_logger(__name__)


if is_vision_available():
      import PIL






class     __magic_name__     (							_snake_case     ):
 UpperCAmelCase				     =    ["""pixel_values"""]


 def __init__(    self		:			str      ,    lowerCAmelCase__		:			bool = True      ,    lowerCAmelCase__		:			Dict[str, int] = None      ,    lowerCAmelCase__		:			PILImageResampling = PILImageResampling.BICUBIC      ,    lowerCAmelCase__		:			bool = True      ,    lowerCAmelCase__		:			Dict[str, int] = None      ,    lowerCAmelCase__		:			bool = True      ,    lowerCAmelCase__		:			Union[int, float] = 1 / 2_5_5      ,    lowerCAmelCase__		:			bool = True      ,    lowerCAmelCase__		:			Optional[Union[float, List[float]]] = None      ,    lowerCAmelCase__		:			Optional[Union[float, List[float]]] = None      ,    lowerCAmelCase__		:			bool = True      ,    **lowerCAmelCase__		:			str      ,    )     ->   None:
   super().__init__(**lowerCAmelCase__     )
   UpperCAmelCase          =	size if size is not None else {"shortest_edge": 2_2_4}
   UpperCAmelCase          =	get_size_dict(lowerCAmelCase__      ,    default_to_square=lowerCAmelCase__     )
   UpperCAmelCase          =	crop_size if crop_size is not None else {"height": 2_2_4, "width": 2_2_4}
   UpperCAmelCase          =	get_size_dict(lowerCAmelCase__     )

   UpperCAmelCase          =	do_resize
   UpperCAmelCase          =	size
   UpperCAmelCase          =	resample
   UpperCAmelCase          =	do_center_crop
   UpperCAmelCase          =	crop_size
   UpperCAmelCase          =	do_rescale
   UpperCAmelCase          =	rescale_factor
   UpperCAmelCase          =	do_normalize
   UpperCAmelCase          =	image_mean if image_mean is not None else OPENAI_CLIP_MEAN
   UpperCAmelCase          =	image_std if image_std is not None else OPENAI_CLIP_STD
   UpperCAmelCase          =	do_convert_rgb


 def _UpperCamelCase						(    self		:			Optional[Any]      ,    lowerCAmelCase__		:			np.ndarray      ,    lowerCAmelCase__		:			Dict[str, int]      ,    lowerCAmelCase__		:			PILImageResampling = PILImageResampling.BICUBIC      ,    lowerCAmelCase__		:			Optional[Union[str, ChannelDimension]] = None      ,    **lowerCAmelCase__		:			Optional[int]      ,    )     ->   np.ndarray:
   UpperCAmelCase          =	get_size_dict(lowerCAmelCase__      ,    default_to_square=lowerCAmelCase__     )
   UpperCAmelCase          =	get_resize_output_image_size(
       lowerCAmelCase__      ,    size=(size["height"], size["width"])      ,    default_to_square=lowerCAmelCase__     )
   return resize(lowerCAmelCase__      ,    size=lowerCAmelCase__      ,    resample=lowerCAmelCase__      ,    data_format=lowerCAmelCase__      ,    **lowerCAmelCase__     )


 def _UpperCamelCase						(    self		:			List[Any]      ,    lowerCAmelCase__		:			np.ndarray      ,    lowerCAmelCase__		:			Dict[str, int]      ,    lowerCAmelCase__		:			Optional[Union[str, ChannelDimension]] = None      ,    **lowerCAmelCase__		:			List[Any]      ,    )     ->   np.ndarray:
   UpperCAmelCase          =	get_size_dict(lowerCAmelCase__     )
   return center_crop(lowerCAmelCase__      ,    size=(size["height"], size["width"])      ,    data_format=lowerCAmelCase__      ,    **lowerCAmelCase__     )


 def _UpperCamelCase						(    self		:			Any      ,    lowerCAmelCase__		:			np.ndarray      ,    lowerCAmelCase__		:			Union[int, float]      ,    lowerCAmelCase__		:			Optional[Union[str, ChannelDimension]] = None      ,    **lowerCAmelCase__		:			List[Any]      ,    )     ->   Optional[int]:
   return rescale(lowerCAmelCase__      ,    scale=lowerCAmelCase__      ,    data_format=lowerCAmelCase__      ,    **lowerCAmelCase__     )


 def _UpperCamelCase						(    self		:			Union[str, Any]      ,    lowerCAmelCase__		:			np.ndarray      ,    lowerCAmelCase__		:			Union[float, List[float]]      ,    lowerCAmelCase__		:			Union[float, List[float]]      ,    lowerCAmelCase__		:			Optional[Union[str, ChannelDimension]] = None      ,    **lowerCAmelCase__		:			List[str]      ,    )     ->   np.ndarray:
   return normalize(lowerCAmelCase__      ,    mean=lowerCAmelCase__      ,    std=lowerCAmelCase__      ,    data_format=lowerCAmelCase__      ,    **lowerCAmelCase__     )


 def _UpperCamelCase						(    self		:			List[str]      ,    lowerCAmelCase__		:			ImageInput      ,    lowerCAmelCase__		:			bool = None      ,    lowerCAmelCase__		:			Dict[str, int] = None      ,    lowerCAmelCase__		:			PILImageResampling = None      ,    lowerCAmelCase__		:			bool = None      ,    lowerCAmelCase__		:			int = None      ,    lowerCAmelCase__		:			bool = None      ,    lowerCAmelCase__		:			float = None      ,    lowerCAmelCase__		:			bool = None      ,    lowerCAmelCase__		:			Optional[Union[float, List[float]]] = None      ,    lowerCAmelCase__		:			Optional[Union[float, List[float]]] = None      ,    lowerCAmelCase__		:			bool = None      ,    lowerCAmelCase__		:			Optional[Union[str, TensorType]] = None      ,    lowerCAmelCase__		:			Optional[ChannelDimension] = ChannelDimension.FIRST      ,    **lowerCAmelCase__		:			int      ,    )     ->   PIL.Image.Image:
   UpperCAmelCase          =	do_resize if do_resize is not None else self.do_resize
   UpperCAmelCase          =	size if size is not None else self.size
   UpperCAmelCase          =	get_size_dict(lowerCAmelCase__      ,    default_to_square=lowerCAmelCase__     )
   UpperCAmelCase          =	resample if resample is not None else self.resample
   UpperCAmelCase          =	do_center_crop if do_center_crop is not None else self.do_center_crop
   UpperCAmelCase          =	crop_size if crop_size is not None else self.crop_size
   UpperCAmelCase          =	get_size_dict(lowerCAmelCase__     )
   UpperCAmelCase          =	do_rescale if do_rescale is not None else self.do_rescale
   UpperCAmelCase          =	rescale_factor if rescale_factor is not None else self.rescale_factor
   UpperCAmelCase          =	do_normalize if do_normalize is not None else self.do_normalize
   UpperCAmelCase          =	image_mean if image_mean is not None else self.image_mean
   UpperCAmelCase          =	image_std if image_std is not None else self.image_std
   UpperCAmelCase          =	do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb

   UpperCAmelCase          =	make_list_of_images(lowerCAmelCase__     )

   if not valid_images(lowerCAmelCase__     ):
     raise ValueError(
         "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
         "torch.Tensor, tf.Tensor or jax.ndarray."     )

   if do_resize and size is None:
     raise ValueError("Size must be specified if do_resize is True."     )

   if do_center_crop and crop_size is None:
     raise ValueError("Crop size must be specified if do_center_crop is True."     )

   if do_rescale and rescale_factor is None:
     raise ValueError("Rescale factor must be specified if do_rescale is True."     )

   if do_normalize and (image_mean is None or image_std is None):
     raise ValueError("Image mean and std must be specified if do_normalize is True."     )

   # PIL RGBA images are converted to RGB
   if do_convert_rgb:
     UpperCAmelCase          =	[convert_to_rgb(lowerCAmelCase__     ) for image in images]

   # All transformations expect numpy arrays.
   UpperCAmelCase          =	[to_numpy_array(lowerCAmelCase__     ) for image in images]

   if do_resize:
     UpperCAmelCase          =	[self.resize(image=lowerCAmelCase__      ,    size=lowerCAmelCase__      ,    resample=lowerCAmelCase__     ) for image in images]

   if do_center_crop:
     UpperCAmelCase          =	[self.center_crop(image=lowerCAmelCase__      ,    size=lowerCAmelCase__     ) for image in images]

   if do_rescale:
     UpperCAmelCase          =	[self.rescale(image=lowerCAmelCase__      ,    scale=lowerCAmelCase__     ) for image in images]

   if do_normalize:
     UpperCAmelCase          =	[self.normalize(image=lowerCAmelCase__      ,    mean=lowerCAmelCase__      ,    std=lowerCAmelCase__     ) for image in images]

   UpperCAmelCase          =	[to_channel_dimension_format(lowerCAmelCase__      ,    lowerCAmelCase__     ) for image in images]

   UpperCAmelCase          =	{"pixel_values": images}
   return BatchFeature(data=lowerCAmelCase__      ,    tensor_type=lowerCAmelCase__     )



