



'''simple docstring'''

from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_DEFAULT_MEAN,
    IMAGENET_DEFAULT_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_batched,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging


__lowerCAmelCase									=   logging.get_logger(__name__)

class 	__magic_name__						(    _UpperCamelCase   ):
							lowerCAmelCase       :    Dict   				=					['pixel_values']







							def __init__(							self					:       Union[str, Any]	,_UpperCAmelCase					:       bool = True	,_UpperCAmelCase					:       Optional[Dict[str, int]] = None	,_UpperCAmelCase					:       PILImageResampling = PILImageResampling.BICUBIC	,_UpperCAmelCase					:       bool = True	,_UpperCAmelCase					:       bool = True	,_UpperCAmelCase					:       Union[int, float] = 1 / 255	,_UpperCAmelCase					:       Dict[str, int] = None	,_UpperCAmelCase					:       bool = True	,_UpperCAmelCase					:       Optional[Union[float, List[float]]] = None	,_UpperCAmelCase					:       Optional[Union[float, List[float]]] = None	,**_UpperCAmelCase					:       int	,):
									super().__init__(**_UpperCAmelCase		)
									_a     :      Union[str, Any]        =							size if size is not None else {'height': 224, 'width': 224}
									_a     :      Optional[int]        =							get_size_dict(_UpperCAmelCase		)
									_a     :      int        =							crop_size if crop_size is not None else {'height': 224, 'width': 224}
									_a     :      int        =							get_size_dict(_UpperCAmelCase	,default_to_square=_UpperCAmelCase	,param_name='crop_size'		)

									_a     :      Any        =							do_resize
									_a     :      str        =							do_rescale
									_a     :      Optional[Any]        =							do_normalize
									_a     :      Optional[Any]        =							do_center_crop
									_a     :      Any        =							crop_size
									_a     :      Optional[Any]        =							size
									_a     :      Optional[int]        =							resample
									_a     :      str        =							rescale_factor
									_a     :      Optional[int]        =							image_mean if image_mean is not None else IMAGENET_DEFAULT_MEAN
									_a     :      int        =							image_std if image_std is not None else IMAGENET_DEFAULT_STD







							def 							__lowercase			(							self					:       List[str]	,_UpperCAmelCase					:       np.ndarray	,_UpperCAmelCase					:       Dict[str, int]	,_UpperCAmelCase					:       PILImageResampling = PILImageResampling.BILINEAR	,_UpperCAmelCase					:       Optional[Union[str, ChannelDimension]] = None	,**_UpperCAmelCase					:       Optional[Any]	,):
									_a     :      Dict        =							get_size_dict(_UpperCAmelCase		)

									if "shortest_edge" in size:
											_a     :      Union[str, Any]        =							get_resize_output_image_size(_UpperCAmelCase	,size=size['shortest_edge']	,default_to_square=_UpperCAmelCase		)
											# size = get_resize_output_image_size(image, size["shortest_edge"], size["longest_edge"])
									elif "height" in size and "width" in size:
											_a     :      str        =							(size['height'], size['width'])
									else:
											raise ValueError(F"""Size must contain 'height' and 'width' keys or 'shortest_edge' key. Got {size.keys()}"""		)
									return resize(_UpperCAmelCase	,size=_UpperCAmelCase	,resample=_UpperCAmelCase	,data_format=_UpperCAmelCase	,**_UpperCAmelCase		)







							def 							__lowercase			(							self					:       List[str]	,_UpperCAmelCase					:       np.ndarray	,_UpperCAmelCase					:       Dict[str, int]	,_UpperCAmelCase					:       Optional[Union[str, ChannelDimension]] = None	,**_UpperCAmelCase					:       Any	,):
									_a     :      Union[str, Any]        =							get_size_dict(_UpperCAmelCase		)
									if "height" not in size or "width" not in size:
											raise ValueError(F"""The `size` parameter must contain the keys (height, width). Got {size.keys()}"""		)
									return center_crop(_UpperCAmelCase	,size=(size['height'], size['width'])	,data_format=_UpperCAmelCase	,**_UpperCAmelCase		)







							def 							__lowercase			(							self					:       Optional[Any]	,_UpperCAmelCase					:       np.ndarray	,_UpperCAmelCase					:       float	,_UpperCAmelCase					:       Optional[Union[str, ChannelDimension]] = None	,**_UpperCAmelCase					:       str		):
									return rescale(_UpperCAmelCase	,scale=_UpperCAmelCase	,data_format=_UpperCAmelCase	,**_UpperCAmelCase		)







							def 							__lowercase			(							self					:       List[Any]	,_UpperCAmelCase					:       np.ndarray	,_UpperCAmelCase					:       Union[float, List[float]]	,_UpperCAmelCase					:       Union[float, List[float]]	,_UpperCAmelCase					:       Optional[Union[str, ChannelDimension]] = None	,**_UpperCAmelCase					:       List[Any]	,):
									return normalize(_UpperCAmelCase	,mean=_UpperCAmelCase	,std=_UpperCAmelCase	,data_format=_UpperCAmelCase	,**_UpperCAmelCase		)







							def 							__lowercase			(							self					:       Optional[int]	,_UpperCAmelCase					:       ImageInput	,_UpperCAmelCase					:       Optional[bool] = None	,_UpperCAmelCase					:       Dict[str, int] = None	,_UpperCAmelCase					:       PILImageResampling = None	,_UpperCAmelCase					:       bool = None	,_UpperCAmelCase					:       int = None	,_UpperCAmelCase					:       Optional[bool] = None	,_UpperCAmelCase					:       Optional[float] = None	,_UpperCAmelCase					:       Optional[bool] = None	,_UpperCAmelCase					:       Optional[Union[float, List[float]]] = None	,_UpperCAmelCase					:       Optional[Union[float, List[float]]] = None	,_UpperCAmelCase					:       Optional[Union[str, TensorType]] = None	,_UpperCAmelCase					:       Union[str, ChannelDimension] = ChannelDimension.FIRST	,**_UpperCAmelCase					:       Any	,):
									_a     :      str        =							do_resize if do_resize is not None else self.do_resize
									_a     :      Optional[Any]        =							do_rescale if do_rescale is not None else self.do_rescale
									_a     :      Optional[int]        =							do_normalize if do_normalize is not None else self.do_normalize
									_a     :      int        =							do_center_crop if do_center_crop is not None else self.do_center_crop
									_a     :      Dict        =							crop_size if crop_size is not None else self.crop_size
									_a     :      Union[str, Any]        =							get_size_dict(_UpperCAmelCase	,param_name='crop_size'	,default_to_square=_UpperCAmelCase		)
									_a     :      str        =							resample if resample is not None else self.resample
									_a     :      Optional[Any]        =							rescale_factor if rescale_factor is not None else self.rescale_factor
									_a     :      Optional[Any]        =							image_mean if image_mean is not None else self.image_mean
									_a     :      Optional[Any]        =							image_std if image_std is not None else self.image_std

									_a     :      Union[str, Any]        =							size if size is not None else self.size
									_a     :      Optional[int]        =							get_size_dict(_UpperCAmelCase		)

									if not is_batched(_UpperCAmelCase		):
											_a     :      Tuple        =							[images]

									if not valid_images(_UpperCAmelCase		):
											raise ValueError(
											    'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
											    'torch.Tensor, tf.Tensor or jax.ndarray.'		)

									if do_resize and size is None:
											raise ValueError('Size must be specified if do_resize is True.'		)

									if do_center_crop and crop_size is None:
											raise ValueError('Crop size must be specified if do_center_crop is True.'		)

									if do_rescale and rescale_factor is None:
											raise ValueError('Rescale factor must be specified if do_rescale is True.'		)

									# All transformations expect numpy arrays.
									_a     :      int        =							[to_numpy_array(_UpperCAmelCase		) for image in images]

									if do_resize:
											_a     :      Dict        =							[self.resize(image=_UpperCAmelCase	,size=_UpperCAmelCase	,resample=_UpperCAmelCase		) for image in images]

									if do_center_crop:
											_a     :      Tuple        =							[self.center_crop(image=_UpperCAmelCase	,size=_UpperCAmelCase		) for image in images]

									if do_rescale:
											_a     :      Optional[Any]        =							[self.rescale(image=_UpperCAmelCase	,scale=_UpperCAmelCase		) for image in images]

									if do_normalize:
											_a     :      str        =							[self.normalize(image=_UpperCAmelCase	,mean=_UpperCAmelCase	,std=_UpperCAmelCase		) for image in images]

									_a     :      Optional[Any]        =							[to_channel_dimension_format(_UpperCAmelCase	,_UpperCAmelCase		) for image in images]

									_a     :      Optional[Any]        =							{'pixel_values': images}
									return BatchFeature(data=_UpperCAmelCase	,tensor_type=_UpperCAmelCase		)









'''simple docstring'''





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase      						=	logging.get_logger(__name__)
_lowerCAmelCase      						=	{'''vocab_file''': '''spiece.model'''}

_lowerCAmelCase      						=	{
    '''vocab_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/spiece.model''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/spiece.model''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/spiece.model''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model''',
    }
}

_lowerCAmelCase      						=	{
    '''albert-base-v1''': 512,
    '''albert-large-v1''': 512,
    '''albert-xlarge-v1''': 512,
    '''albert-xxlarge-v1''': 512,
    '''albert-base-v2''': 512,
    '''albert-large-v2''': 512,
    '''albert-xlarge-v2''': 512,
    '''albert-xxlarge-v2''': 512,
}

_lowerCAmelCase      						=	'''▁'''






class        lowerCAmelCase_(				SCREAMING_SNAKE_CASE_     ):
  '''simple docstring'''




  __lowercase							:      Any				=      VOCAB_FILES_NAMES
  __lowercase							:      Optional[Any]				=      PRETRAINED_VOCAB_FILES_MAP
  __lowercase							:      int				=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




  def __init__(    self  ,__UpperCAmelCase  ,__UpperCAmelCase=True  ,__UpperCAmelCase=True  ,__UpperCAmelCase=False  ,__UpperCAmelCase="[CLS]"  ,__UpperCAmelCase="[SEP]"  ,__UpperCAmelCase="<unk>"  ,__UpperCAmelCase="[SEP]"  ,__UpperCAmelCase="<pad>"  ,__UpperCAmelCase="[CLS]"  ,__UpperCAmelCase="[MASK]"  ,__UpperCAmelCase = None  ,**__UpperCAmelCase  ,)    ->		None:
      # Mask token behave like a normal word, i.e. include the space before it and
      # is included in the raw text, there should be a match in a non-normalized sentence.
      lowerCAmelCase__ :  Tuple							      =					(
          AddedToken(__UpperCAmelCase  ,lstrip=__UpperCAmelCase  ,rstrip=__UpperCAmelCase  ,normalized=__UpperCAmelCase      )
          if isinstance(__UpperCAmelCase  ,__UpperCAmelCase      )
          else mask_token
      )

      lowerCAmelCase__ :  Any							      =					{} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          do_lower_case=__UpperCAmelCase  ,remove_space=__UpperCAmelCase  ,keep_accents=__UpperCAmelCase  ,bos_token=__UpperCAmelCase  ,eos_token=__UpperCAmelCase  ,unk_token=__UpperCAmelCase  ,sep_token=__UpperCAmelCase  ,pad_token=__UpperCAmelCase  ,cls_token=__UpperCAmelCase  ,mask_token=__UpperCAmelCase  ,sp_model_kwargs=self.sp_model_kwargs  ,**__UpperCAmelCase  ,)

      lowerCAmelCase__ :  str							      =					do_lower_case
      lowerCAmelCase__ :  int							      =					remove_space
      lowerCAmelCase__ :  Tuple							      =					keep_accents
      lowerCAmelCase__ :  Any							      =					vocab_file

      lowerCAmelCase__ :  Tuple							      =					spm.SentencePieceProcessor(**self.sp_model_kwargs      )
      self.sp_model.Load(__UpperCAmelCase      )




  @property
  def 							UpperCAmelCase_	(    self      )    ->		Optional[int]:
      return len(self.sp_model      )




  def 							UpperCAmelCase_	(    self      )    ->		Optional[Any]:
      lowerCAmelCase__ :  Union[str, Any]							      =					{self.convert_ids_to_tokens(__UpperCAmelCase      ): i for i in range(self.vocab_size      )}
      vocab.update(self.added_tokens_encoder      )
      return vocab




  def __getstate__(    self      )    ->		Any:
      lowerCAmelCase__ :  Optional[Any]							      =					self.__dict__.copy()
      lowerCAmelCase__ :  Optional[Any]							      =					None
      return state




  def __setstate__(    self  ,__UpperCAmelCase      )    ->		List[Any]:
      lowerCAmelCase__ :  List[str]							      =					d

      # for backward compatibility
      if not hasattr(self  ,"""sp_model_kwargs"""      ):
          lowerCAmelCase__ :  Union[str, Any]							      =					{}

      lowerCAmelCase__ :  Any							      =					spm.SentencePieceProcessor(**self.sp_model_kwargs      )
      self.sp_model.Load(self.vocab_file      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase      )    ->		Optional[int]:
      if self.remove_space:
          lowerCAmelCase__ :  int							      =					""" """.join(inputs.strip().split()      )
      else:
          lowerCAmelCase__ :  str							      =					inputs
      lowerCAmelCase__ :  Tuple							      =					outputs.replace("""``"""  ,"""\""""      ).replace("""''"""  ,"""\""""      )

      if not self.keep_accents:
          lowerCAmelCase__ :  Any							      =					unicodedata.normalize("""NFKD"""  ,__UpperCAmelCase      )
          lowerCAmelCase__ :  Dict							      =					"""""".join([c for c in outputs if not unicodedata.combining(__UpperCAmelCase      )]      )
      if self.do_lower_case:
          lowerCAmelCase__ :  Tuple							      =					outputs.lower()

      return outputs




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase      )    ->		List[str]:
      lowerCAmelCase__ :  List[str]							      =					self.preprocess_text(__UpperCAmelCase      )
      lowerCAmelCase__ :  Optional[int]							      =					self.sp_model.encode(__UpperCAmelCase  ,out_type=__UpperCAmelCase      )
      lowerCAmelCase__ :  str							      =					[]
      for piece in pieces:
          if len(__UpperCAmelCase      ) > 1 and piece[-1] == str(""","""      ) and piece[-2].isdigit():
              lowerCAmelCase__ :  List[Any]							      =					self.sp_model.EncodeAsPieces(piece[:-1].replace(__UpperCAmelCase  ,""""""      )      )
              if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                  if len(cur_pieces[0]      ) == 1:
                      lowerCAmelCase__ :  str							      =					cur_pieces[1:]
                  else:
                      lowerCAmelCase__ :  int							      =					cur_pieces[0][1:]
              cur_pieces.append(piece[-1]      )
              new_pieces.extend(__UpperCAmelCase      )
          else:
              new_pieces.append(__UpperCAmelCase      )

      return new_pieces




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase      )    ->		List[str]:
      return self.sp_model.PieceToId(__UpperCAmelCase      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase      )    ->		Any:
      return self.sp_model.IdToPiece(__UpperCAmelCase      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase      )    ->		str:
      lowerCAmelCase__ :  str							      =					[]
      lowerCAmelCase__ :  Tuple							      =					""""""
      lowerCAmelCase__ :  Tuple							      =					False
      for token in tokens:
          # make sure that special tokens are not decoded using sentencepiece model
          if token in self.all_special_tokens:
              if not prev_is_special:
                  out_string += " "
              out_string += self.sp_model.decode(__UpperCAmelCase      ) + token
              lowerCAmelCase__ :  Union[str, Any]							      =					True
              lowerCAmelCase__ :  List[str]							      =					[]
          else:
              current_sub_tokens.append(__UpperCAmelCase      )
              lowerCAmelCase__ :  Optional[Any]							      =					False
      out_string += self.sp_model.decode(__UpperCAmelCase      )
      return out_string.strip()




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase = None      )    ->		List[int]:
      lowerCAmelCase__ :  int							      =					[self.sep_token_id]
      lowerCAmelCase__ :  Dict							      =					[self.cls_token_id]
      if token_ids_a is None:
          return cls + token_ids_a + sep
      return cls + token_ids_a + sep + token_ids_a + sep




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase = None  ,__UpperCAmelCase = False      )    ->		List[int]:

      if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=__UpperCAmelCase  ,token_ids_a=__UpperCAmelCase  ,already_has_special_tokens=__UpperCAmelCase      )

      if token_ids_a is not None:
          return [1] + ([0] * len(__UpperCAmelCase      )) + [1] + ([0] * len(__UpperCAmelCase      )) + [1]
      return [1] + ([0] * len(__UpperCAmelCase      )) + [1]




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase = None      )    ->		List[int]:
      lowerCAmelCase__ :  List[str]							      =					[self.sep_token_id]
      lowerCAmelCase__ :  Tuple							      =					[self.cls_token_id]

      if token_ids_a is None:
          return len(cls + token_ids_a + sep      ) * [0]
      return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]






  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase = None      )    ->		Tuple[str]:
      if not os.path.isdir(__UpperCAmelCase      ):
          logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""      )
          return
      lowerCAmelCase__ :  int							      =					os.path.join(
          __UpperCAmelCase  ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]      )

      if os.path.abspath(self.vocab_file      ) != os.path.abspath(__UpperCAmelCase      ) and os.path.isfile(self.vocab_file      ):
          copyfile(self.vocab_file  ,__UpperCAmelCase      )
      elif not os.path.isfile(self.vocab_file      ):
          with open(__UpperCAmelCase  ,"""wb"""      ) as fi:
              lowerCAmelCase__ :  List[Any]							      =					self.sp_model.serialized_model_proto()
              fi.write(__UpperCAmelCase      )

      return (out_vocab_file,)


