

"""simple docstring"""





from typing import Dict, List, Optional, Union

import numpy as np

from transformers.utils import is_vision_available
from transformers.utils.generic import TensorType

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_valid_image,
    to_numpy_array,
    valid_images,
)
from ...utils import logging


if is_vision_available():
							import PIL

__UpperCamelCase  :      Any							      =		logging.get_logger(__name__)

def   __SCREAMING_SNAKE_CASE  (      A_ ):
	if isinstance(A_						,     (list, tuple) ) and isinstance(videos[0]						,     (list, tuple) ) and is_valid_image(videos[0][0] ):
		return videos

	elif isinstance(A_						,     (list, tuple) ) and is_valid_image(videos[0] ):
		return [videos]

	elif is_valid_image(A_ ):
		return [[videos]]

	raise ValueError(f'Could not make batched video from {videos}' )







class 				SCREAMING_SNAKE_CASE					(		a_       ):






					"""simple docstring"""



					lowercase__				 =       ["pixel_values"]







					def __init__(       self			:				Tuple   ,lowercase_			:				bool = True   ,lowercase_			:				Dict[str, int] = None   ,lowercase_			:				PILImageResampling = PILImageResampling.BILINEAR   ,lowercase_			:				bool = True   ,lowercase_			:				Dict[str, int] = None   ,lowercase_			:				bool = True   ,lowercase_			:				Union[int, float] = 1 / 2_5_5   ,lowercase_			:				bool = True   ,lowercase_			:				bool = True   ,lowercase_			:				Optional[Union[float, List[float]]] = None   ,lowercase_			:				Optional[Union[float, List[float]]] = None   ,**lowercase_			:				str   ,):
						super().__init__(**lowercase_		)
						lowerCAmelCase__	:						Union[str, Any]  				=	size if size is not None else {'''shortest_edge''': 2_5_6}
						lowerCAmelCase__	:						Any  				=	get_size_dict(lowercase_   ,default_to_square=lowercase_		)
						lowerCAmelCase__	:						Dict  				=	crop_size if crop_size is not None else {'''height''': 2_2_4, '''width''': 2_2_4}
						lowerCAmelCase__	:						List[str]  				=	get_size_dict(lowercase_   ,param_name='''crop_size'''		)

						lowerCAmelCase__	:						Union[str, Any]  				=	do_resize
						lowerCAmelCase__	:						Optional[Any]  				=	size
						lowerCAmelCase__	:						Any  				=	do_center_crop
						lowerCAmelCase__	:						int  				=	crop_size
						lowerCAmelCase__	:						List[Any]  				=	resample
						lowerCAmelCase__	:						Tuple  				=	do_rescale
						lowerCAmelCase__	:						List[Any]  				=	rescale_factor
						lowerCAmelCase__	:						str  				=	offset
						lowerCAmelCase__	:						Any  				=	do_normalize
						lowerCAmelCase__	:						Tuple  				=	image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
						lowerCAmelCase__	:						Optional[Any]  				=	image_std if image_std is not None else IMAGENET_STANDARD_STD







					def        __lowerCAmelCase (       self			:				Tuple   ,lowercase_			:				np.ndarray   ,lowercase_			:				Dict[str, int]   ,lowercase_			:				PILImageResampling = PILImageResampling.BILINEAR   ,lowercase_			:				Optional[Union[str, ChannelDimension]] = None   ,**lowercase_			:				Optional[Any]   ,):
						lowerCAmelCase__	:						Any  				=	get_size_dict(lowercase_   ,default_to_square=lowercase_		)
						if "shortest_edge" in size:
							lowerCAmelCase__	:						Any  				=	get_resize_output_image_size(lowercase_   ,size['''shortest_edge''']   ,default_to_square=lowercase_		)
						elif "height" in size and "width" in size:
							lowerCAmelCase__	:						Dict  				=	(size['''height'''], size['''width'''])
						else:
							raise ValueError(F'Size must have \'height\' and \'width\' or \'shortest_edge\' as keys. Got {size.keys()}'		)
						return resize(lowercase_   ,size=lowercase_   ,resample=lowercase_   ,data_format=lowercase_   ,**lowercase_		)







					def        __lowerCAmelCase (       self			:				Optional[int]   ,lowercase_			:				np.ndarray   ,lowercase_			:				Dict[str, int]   ,lowercase_			:				Optional[Union[str, ChannelDimension]] = None   ,**lowercase_			:				Dict   ,):
						lowerCAmelCase__	:						List[Any]  				=	get_size_dict(lowercase_		)
						if "height" not in size or "width" not in size:
							raise ValueError(F'Size must have \'height\' and \'width\' as keys. Got {size.keys()}'		)
						return center_crop(lowercase_   ,size=(size['''height'''], size['''width'''])   ,data_format=lowercase_   ,**lowercase_		)







					def        __lowerCAmelCase (       self			:				int   ,lowercase_			:				np.ndarray   ,lowercase_			:				Union[int, float]   ,lowercase_			:				bool = True   ,lowercase_			:				Optional[Union[str, ChannelDimension]] = None   ,**lowercase_			:				Dict   ,):
						lowerCAmelCase__	:						int  				=	image.astype(np.floataa		)
						if offset:
							lowerCAmelCase__	:						Tuple  				=	image - (scale / 2)
						return rescale(lowercase_   ,scale=lowercase_   ,data_format=lowercase_   ,**lowercase_		)







					def        __lowerCAmelCase (       self			:				Any   ,lowercase_			:				np.ndarray   ,lowercase_			:				Union[float, List[float]]   ,lowercase_			:				Union[float, List[float]]   ,lowercase_			:				Optional[Union[str, ChannelDimension]] = None   ,**lowercase_			:				Dict   ,):
						return normalize(lowercase_   ,mean=lowercase_   ,std=lowercase_   ,data_format=lowercase_   ,**lowercase_		)







					def        __lowerCAmelCase (       self			:				Tuple   ,lowercase_			:				ImageInput   ,lowercase_			:				bool = None   ,lowercase_			:				Dict[str, int] = None   ,lowercase_			:				PILImageResampling = None   ,lowercase_			:				bool = None   ,lowercase_			:				Dict[str, int] = None   ,lowercase_			:				bool = None   ,lowercase_			:				float = None   ,lowercase_			:				bool = None   ,lowercase_			:				bool = None   ,lowercase_			:				Optional[Union[float, List[float]]] = None   ,lowercase_			:				Optional[Union[float, List[float]]] = None   ,lowercase_			:				Optional[ChannelDimension] = ChannelDimension.FIRST   ,):
						if do_resize and size is None or resample is None:
							raise ValueError('''Size and resample must be specified if do_resize is True.'''		)

						if do_center_crop and crop_size is None:
							raise ValueError('''Crop size must be specified if do_center_crop is True.'''		)

						if do_rescale and rescale_factor is None:
							raise ValueError('''Rescale factor must be specified if do_rescale is True.'''		)

						if do_normalize and (image_mean is None or image_std is None):
							raise ValueError('''Image mean and std must be specified if do_normalize is True.'''		)

						if offset and not do_rescale:
							raise ValueError('''For offset, do_rescale must also be set to True.'''		)

						# All transformations expect numpy arrays.
						lowerCAmelCase__	:						List[Any]  				=	to_numpy_array(lowercase_		)

						if do_resize:
							lowerCAmelCase__	:						Optional[int]  				=	self.resize(image=lowercase_   ,size=lowercase_   ,resample=lowercase_		)

						if do_center_crop:
							lowerCAmelCase__	:						Union[str, Any]  				=	self.center_crop(lowercase_   ,size=lowercase_		)

						if do_rescale:
							lowerCAmelCase__	:						Any  				=	self.rescale(image=lowercase_   ,scale=lowercase_   ,offset=lowercase_		)

						if do_normalize:
							lowerCAmelCase__	:						List[str]  				=	self.normalize(image=lowercase_   ,mean=lowercase_   ,std=lowercase_		)

						lowerCAmelCase__	:						Optional[Any]  				=	to_channel_dimension_format(lowercase_   ,lowercase_		)
						return image







					def        __lowerCAmelCase (       self			:				Tuple   ,lowercase_			:				ImageInput   ,lowercase_			:				bool = None   ,lowercase_			:				Dict[str, int] = None   ,lowercase_			:				PILImageResampling = None   ,lowercase_			:				bool = None   ,lowercase_			:				Dict[str, int] = None   ,lowercase_			:				bool = None   ,lowercase_			:				float = None   ,lowercase_			:				bool = None   ,lowercase_			:				bool = None   ,lowercase_			:				Optional[Union[float, List[float]]] = None   ,lowercase_			:				Optional[Union[float, List[float]]] = None   ,lowercase_			:				Optional[Union[str, TensorType]] = None   ,lowercase_			:				ChannelDimension = ChannelDimension.FIRST   ,**lowercase_			:				Optional[int]   ,):
						lowerCAmelCase__	:						int  				=	do_resize if do_resize is not None else self.do_resize
						lowerCAmelCase__	:						str  				=	resample if resample is not None else self.resample
						lowerCAmelCase__	:						Optional[int]  				=	do_center_crop if do_center_crop is not None else self.do_center_crop
						lowerCAmelCase__	:						Optional[Any]  				=	do_rescale if do_rescale is not None else self.do_rescale
						lowerCAmelCase__	:						List[str]  				=	rescale_factor if rescale_factor is not None else self.rescale_factor
						lowerCAmelCase__	:						str  				=	offset if offset is not None else self.offset
						lowerCAmelCase__	:						Tuple  				=	do_normalize if do_normalize is not None else self.do_normalize
						lowerCAmelCase__	:						int  				=	image_mean if image_mean is not None else self.image_mean
						lowerCAmelCase__	:						Optional[Any]  				=	image_std if image_std is not None else self.image_std

						lowerCAmelCase__	:						Optional[int]  				=	size if size is not None else self.size
						lowerCAmelCase__	:						Optional[int]  				=	get_size_dict(lowercase_   ,default_to_square=lowercase_		)
						lowerCAmelCase__	:						Any  				=	crop_size if crop_size is not None else self.crop_size
						lowerCAmelCase__	:						Dict  				=	get_size_dict(lowercase_   ,param_name='''crop_size'''		)

						if not valid_images(lowercase_		):
							raise ValueError(
							    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
							    '''torch.Tensor, tf.Tensor or jax.ndarray.'''		)

						lowerCAmelCase__	:						Tuple  				=	make_batched(lowercase_		)

						lowerCAmelCase__	:						Optional[int]  				=	[
						    [
						        self._preprocess_image(
						            image=lowercase_   ,do_resize=lowercase_   ,size=lowercase_   ,resample=lowercase_   ,do_center_crop=lowercase_   ,crop_size=lowercase_   ,do_rescale=lowercase_   ,rescale_factor=lowercase_   ,offset=lowercase_   ,do_normalize=lowercase_   ,image_mean=lowercase_   ,image_std=lowercase_   ,data_format=lowercase_   ,)
						        for img in video
						    ]
						    for video in videos
						]

						lowerCAmelCase__	:						Dict  				=	{'''pixel_values''': videos}
						return BatchFeature(data=lowercase_   ,tensor_type=lowercase_		)











"""simple docstring"""





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE__					=   "▁"

SCREAMING_SNAKE_CASE__					=   {"vocab_file": "spiece.model"}

SCREAMING_SNAKE_CASE__					=   {
    "vocab_file": {"google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"}
}

SCREAMING_SNAKE_CASE__					=   {
    "google/pegasus-xsum": 512,
}


SCREAMING_SNAKE_CASE__					=   logging.get_logger(__name__)

class 		lowercase					(					_UpperCAmelCase					):
      _SCREAMING_SNAKE_CASE					=      VOCAB_FILES_NAMES

      _SCREAMING_SNAKE_CASE					=      VOCAB_FILES_NAMES
      _SCREAMING_SNAKE_CASE					=      PRETRAINED_VOCAB_FILES_MAP
      _SCREAMING_SNAKE_CASE					=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _SCREAMING_SNAKE_CASE					=      ['input_ids', 'attention_mask']





      def __init__(       self      ,						lowercase      ,						lowercase="<pad>"      ,						lowercase="</s>"      ,						lowercase="<unk>"      ,						lowercase="<mask_2>"      ,						lowercase="<mask_1>"      ,						lowercase=None      ,						lowercase=103      ,						lowercase = None      ,						**lowercase      ,						)		->     None:
            lowerCAmelCase  						=       offset
            if additional_special_tokens is not None:
                  if not isinstance(lowercase      ,						lowercase ):
                        raise TypeError(
                            f'additional_special_tokens should be of type {type(lowercase )}, but is'
                            f' {type(lowercase )}' )

                  lowerCAmelCase  						=       (
                      ([mask_token_sent] + additional_special_tokens)
                      if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
                      else additional_special_tokens
                  )
                  # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
                  additional_special_tokens_extended += [
                      f'<unk_{i}>' for i in range(len(lowercase )      ,						self.offset - 1 )
                  ]

                  if len(set(lowercase ) ) != len(lowercase ):
                        raise ValueError(
                            """Please make sure that the provided additional_special_tokens do not contain an incorrectly"""
                            f' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.' )
                  lowerCAmelCase  						=       additional_special_tokens_extended
            else:
                  lowerCAmelCase  						=       [mask_token_sent] if mask_token_sent is not None else []
                  additional_special_tokens += [f'<unk_{i}>' for i in range(2      ,						self.offset )]

            lowerCAmelCase  						=       {} if sp_model_kwargs is None else sp_model_kwargs

            super().__init__(
                eos_token=lowercase      ,						unk_token=lowercase      ,						mask_token=lowercase      ,						pad_token=lowercase      ,						mask_token_sent=lowercase      ,						offset=lowercase      ,						additional_special_tokens=lowercase      ,						sp_model_kwargs=self.sp_model_kwargs      ,						**lowercase      ,						)
            lowerCAmelCase  						=       mask_token_sent
            lowerCAmelCase  						=       vocab_file
            lowerCAmelCase  						=       spm.SentencePieceProcessor(**self.sp_model_kwargs )
            self.sp_model.Load(lowercase )

            # add special tokens to encoder dict
            lowerCAmelCase  						=       {
                0: self.pad_token,
                1: self.eos_token,
            }

            if self.mask_token_sent is not None:
                  self.encoder.update(
                      {
                          2: self.mask_token_sent,
                          3: self.mask_token,
                      } )

            if self.offset > 0:
                  # entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
                  # mask_token_sent is already added to list -> so start at 1
                  self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1      ,						self.offset - 1 )} )

            lowerCAmelCase  						=       {v: k for k, v in self.encoder.items()}





      @property
      def     _snake_case			(       self )		->     int:
            return len(self.sp_model ) + self.offset





      def     _snake_case			(       self )		->     Dict[str, int]:
            lowerCAmelCase  						=       {self.convert_ids_to_tokens(lowercase ): i for i in range(self.vocab_size )}
            vocab.update(self.added_tokens_encoder )
            return vocab





      def __getstate__(       self )		->     Optional[int]:
            lowerCAmelCase  						=       self.__dict__.copy()
            lowerCAmelCase  						=       None
            return state





      def __setstate__(       self      ,						lowercase )		->     List[Any]:
            lowerCAmelCase  						=       d

            # for backward compatibility
            if not hasattr(self      ,						"""sp_model_kwargs""" ):
                  lowerCAmelCase  						=       {}

            lowerCAmelCase  						=       spm.SentencePieceProcessor(**self.sp_model_kwargs )
            self.sp_model.Load(self.vocab_file )





      def     _snake_case			(       self      ,						lowercase )		->     List[str]:
            return self.sp_model.encode(lowercase      ,						out_type=lowercase )





      def     _snake_case			(       self      ,						lowercase )		->     int:
            if token in self.decoder:
                  return self.decoder[token]
            elif token in self.added_tokens_decoder:
                  return self.added_tokens_decoder[token]
            lowerCAmelCase  						=       self.sp_model.piece_to_id(lowercase )
            return sp_id + self.offset





      def     _snake_case			(       self      ,						lowercase )		->     str:
            if index in self.encoder:
                  return self.encoder[index]
            elif index in self.added_tokens_encoder:
                  return self.added_tokens_encoder[index]
            else:
                  lowerCAmelCase  						=       self.sp_model.IdToPiece(index - self.offset )
            return token





      def     _snake_case			(       self      ,						lowercase )		->     Optional[int]:
            lowerCAmelCase  						=       []
            lowerCAmelCase  						=       """"""
            for token in tokens:
                  # make sure that special tokens are not decoded using sentencepiece model
                  if token in self.all_special_tokens:
                        out_string += self.sp_model.decode(lowercase ) + token
                        lowerCAmelCase  						=       []
                  else:
                        current_sub_tokens.append(lowercase )
            out_string += self.sp_model.decode(lowercase )
            return out_string.strip()





      def     _snake_case			(       self      ,						lowercase=False )		->     Tuple:
            return 1





      def     _snake_case			(       self      ,						lowercase )		->     Tuple:
            lowerCAmelCase  						=       set(self.all_special_ids )  # call it once instead of inside list comp
            all_special_ids.remove(self.unk_token_id )  # <unk> is only sometimes special

            return [1 if x in all_special_ids else 0 for x in seq]





      def     _snake_case			(       self      ,						lowercase      ,						lowercase = None      ,						lowercase = False )		->     List[int]:
            if already_has_special_tokens:
                  return self._special_token_mask(lowercase )
            elif token_ids_a is None:
                  return self._special_token_mask(lowercase ) + [1]
            else:
                  return self._special_token_mask(token_ids_a + token_ids_a ) + [1]





      def     _snake_case			(       self      ,						lowercase      ,						lowercase=None )		->     List[int]:
            if token_ids_a is None:
                  return token_ids_a + [self.eos_token_id]
            # We don't expect to process pairs, but leave the pair logic for API consistency
            return token_ids_a + token_ids_a + [self.eos_token_id]





      def     _snake_case			(       self      ,						lowercase      ,						lowercase = None )		->     Tuple[str]:
            if not os.path.isdir(lowercase ):
                  logger.error(f'Vocabulary path ({save_directory}) should be a directory' )
                  return
            lowerCAmelCase  						=       os.path.join(
                lowercase      ,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )

            if os.path.abspath(self.vocab_file ) != os.path.abspath(lowercase ) and os.path.isfile(self.vocab_file ):
                  copyfile(self.vocab_file      ,						lowercase )
            elif not os.path.isfile(self.vocab_file ):
                  with open(lowercase      ,						"""wb""" ) as fi:
                        lowerCAmelCase  						=       self.sp_model.serialized_model_proto()
                        fi.write(lowercase )

            return (out_vocab_file,)






