






import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase								=						logging.get_logger(__name__)
__UpperCAmelCase								=						{'vocab_file': 'spiece.model'}

__UpperCAmelCase								=						{
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

__UpperCAmelCase								=						{
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

__UpperCAmelCase								=						'▁'



class   lowerCamelCase      (_snake_case			):







		'''simple docstring'''



		_snake_case    :    Union[str, Any]       						=  VOCAB_FILES_NAMES
		_snake_case    :    List[str]       						=  PRETRAINED_VOCAB_FILES_MAP
		_snake_case    :    Tuple       						=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



		def __init__(							self				,		_UpperCamelCase				,		_UpperCamelCase=True				,		_UpperCamelCase=True				,		_UpperCamelCase=False				,		_UpperCamelCase="[CLS]"				,		_UpperCamelCase="[SEP]"				,		_UpperCamelCase="<unk>"				,		_UpperCamelCase="[SEP]"				,		_UpperCamelCase="<pad>"				,		_UpperCamelCase="[CLS]"				,		_UpperCamelCase="[MASK]"				,		_UpperCamelCase = None				,		**_UpperCamelCase				,		)	->							None:
							# Mask token behave like a normal word, i.e. include the space before it and
							# is included in the raw text, there should be a match in a non-normalized sentence.
							UpperCAmelCase_    :      Dict            =						(
							    AddedToken(_UpperCamelCase				,		lstrip=_UpperCamelCase				,		rstrip=_UpperCamelCase				,		normalized=_UpperCamelCase   )
							    if isinstance(_UpperCamelCase				,		_UpperCamelCase   )
							    else mask_token
							)

							UpperCAmelCase_    :      Tuple            =						{} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    do_lower_case=_UpperCamelCase				,		remove_space=_UpperCamelCase				,		keep_accents=_UpperCamelCase				,		bos_token=_UpperCamelCase				,		eos_token=_UpperCamelCase				,		unk_token=_UpperCamelCase				,		sep_token=_UpperCamelCase				,		pad_token=_UpperCamelCase				,		cls_token=_UpperCamelCase				,		mask_token=_UpperCamelCase				,		sp_model_kwargs=self.sp_model_kwargs				,		**_UpperCamelCase				,		)

							UpperCAmelCase_    :      List[str]            =						do_lower_case
							UpperCAmelCase_    :      Optional[Any]            =						remove_space
							UpperCAmelCase_    :      Union[str, Any]            =						keep_accents
							UpperCAmelCase_    :      Any            =						vocab_file

							UpperCAmelCase_    :      str            =						spm.SentencePieceProcessor(**self.sp_model_kwargs   )
							self.sp_model.Load(_UpperCamelCase   )



		@property
		def 					__UpperCAmelCase			(							self   )	->							Union[str, Any]:
							return len(self.sp_model   )



		def 					__UpperCAmelCase			(							self   )	->							Optional[Any]:
							UpperCAmelCase_    :      Optional[Any]            =						{self.convert_ids_to_tokens(_UpperCamelCase   ): i for i in range(self.vocab_size   )}
							vocab.update(self.added_tokens_encoder   )
							return vocab



		def __getstate__(							self   )	->							Union[str, Any]:
							UpperCAmelCase_    :      List[str]            =						self.__dict__.copy()
							UpperCAmelCase_    :      Any            =						None
							return state



		def __setstate__(							self				,		_UpperCamelCase   )	->							Union[str, Any]:
							UpperCAmelCase_    :      Optional[Any]            =						d

							# for backward compatibility
							if not hasattr(self				,		'sp_model_kwargs'   ):
												UpperCAmelCase_    :      Dict            =						{}

							UpperCAmelCase_    :      Optional[Any]            =						spm.SentencePieceProcessor(**self.sp_model_kwargs   )
							self.sp_model.Load(self.vocab_file   )



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase   )	->							List[str]:
							if self.remove_space:
												UpperCAmelCase_    :      Tuple            =						' '.join(inputs.strip().split()   )
							else:
												UpperCAmelCase_    :      List[Any]            =						inputs
							UpperCAmelCase_    :      int            =						outputs.replace('``'				,		'"'   ).replace('\'\''				,		'"'   )

							if not self.keep_accents:
												UpperCAmelCase_    :      int            =						unicodedata.normalize('NFKD'				,		_UpperCamelCase   )
												UpperCAmelCase_    :      str            =						''.join([c for c in outputs if not unicodedata.combining(_UpperCamelCase   )]   )
							if self.do_lower_case:
												UpperCAmelCase_    :      Optional[int]            =						outputs.lower()

							return outputs



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase   )	->							List[str]:
							UpperCAmelCase_    :      Any            =						self.preprocess_text(_UpperCamelCase   )
							UpperCAmelCase_    :      int            =						self.sp_model.encode(_UpperCamelCase				,		out_type=_UpperCamelCase   )
							UpperCAmelCase_    :      Any            =						[]
							for piece in pieces:
												if len(_UpperCamelCase   ) > 1 and piece[-1] == str(','   ) and piece[-2].isdigit():
																	UpperCAmelCase_    :      List[str]            =						self.sp_model.EncodeAsPieces(piece[:-1].replace(_UpperCamelCase				,		''   )   )
																	if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																						if len(cur_pieces[0]   ) == 1:
																											UpperCAmelCase_    :      Union[str, Any]            =						cur_pieces[1:]
																						else:
																											UpperCAmelCase_    :      str            =						cur_pieces[0][1:]
																	cur_pieces.append(piece[-1]   )
																	new_pieces.extend(_UpperCamelCase   )
												else:
																	new_pieces.append(_UpperCamelCase   )

							return new_pieces



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase   )	->							Any:
							return self.sp_model.PieceToId(_UpperCamelCase   )



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase   )	->							Tuple:
							return self.sp_model.IdToPiece(_UpperCamelCase   )



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase   )	->							Optional[Any]:
							UpperCAmelCase_    :      Any            =						[]
							UpperCAmelCase_    :      List[Any]            =						''
							UpperCAmelCase_    :      List[str]            =						False
							for token in tokens:
												# make sure that special tokens are not decoded using sentencepiece model
												if token in self.all_special_tokens:
																	if not prev_is_special:
																						out_string += " "
																	out_string += self.sp_model.decode(_UpperCamelCase   ) + token
																	UpperCAmelCase_    :      Tuple            =						True
																	UpperCAmelCase_    :      Optional[int]            =						[]
												else:
																	current_sub_tokens.append(_UpperCamelCase   )
																	UpperCAmelCase_    :      Optional[Any]            =						False
							out_string += self.sp_model.decode(_UpperCamelCase   )
							return out_string.strip()



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase = None   )	->							List[int]:
							UpperCAmelCase_    :      Tuple            =						[self.sep_token_id]
							UpperCAmelCase_    :      Union[str, Any]            =						[self.cls_token_id]
							if token_ids_a is None:
												return cls + token_ids_a + sep
							return cls + token_ids_a + sep + token_ids_a + sep



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase = None				,		_UpperCamelCase = False   )	->							List[int]:

							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=_UpperCamelCase				,		token_ids_a=_UpperCamelCase				,		already_has_special_tokens=_UpperCamelCase   )

							if token_ids_a is not None:
												return [1] + ([0] * len(_UpperCamelCase   )) + [1] + ([0] * len(_UpperCamelCase   )) + [1]
							return [1] + ([0] * len(_UpperCamelCase   )) + [1]



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase = None   )	->							List[int]:
							UpperCAmelCase_    :      int            =						[self.sep_token_id]
							UpperCAmelCase_    :      List[Any]            =						[self.cls_token_id]

							if token_ids_a is None:
												return len(cls + token_ids_a + sep   ) * [0]
							return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase = None   )	->							Tuple[str]:
							if not os.path.isdir(_UpperCamelCase   ):
												logger.error(f"Vocabulary path ({save_directory}) should be a directory"   )
												return
							UpperCAmelCase_    :      int            =						os.path.join(
							    _UpperCamelCase				,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )

							if os.path.abspath(self.vocab_file   ) != os.path.abspath(_UpperCamelCase   ) and os.path.isfile(self.vocab_file   ):
												copyfile(self.vocab_file				,		_UpperCamelCase   )
							elif not os.path.isfile(self.vocab_file   ):
												with open(_UpperCamelCase				,		'wb'   ) as fi:
																	UpperCAmelCase_    :      Tuple            =						self.sp_model.serialized_model_proto()
																	fi.write(_UpperCamelCase   )

							return (out_vocab_file,)














import warnings

from ...utils import logging
from .image_processing_imagegpt import ImageGPTImageProcessor


__UpperCAmelCase								=						logging.get_logger(__name__)



class   lowerCamelCase      (_snake_case			):







		'''simple docstring'''



		def __init__(							self				,		*_UpperCamelCase				,		**_UpperCamelCase   )	->							None:
							warnings.warn(
							    'The class ImageGPTFeatureExtractor is deprecated and will be removed in version 5 of Transformers.'
							    ' Please use ImageGPTImageProcessor instead.'				,		_UpperCamelCase				,		)
							super().__init__(*_UpperCamelCase				,		**_UpperCamelCase   )






