
'''simple docstring'''







import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


lowercase    :							Dict      =					logging.get_logger(__name__)

lowercase    :							List[Any]      =					{"vocab_file": "spiece.model"}

lowercase    :							Union[str, Any]      =					{
    "vocab_file": {
        "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model",
        "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model",
    }
}

lowercase    :							List[Any]      =					{
    "xlnet-base-cased": None,
    "xlnet-large-cased": None,
}

# Segments (not really needed)
lowercase    :							str      =					0
lowercase    :							str      =					1
lowercase    :							List[str]      =					2
lowercase    :							Optional[Any]      =					3
lowercase    :							List[Any]      =					4


class 						__UpperCAmelCase      (					_lowerCamelCase						):
							__lowercase      		=       VOCAB_FILES_NAMES
							__lowercase      		=       PRETRAINED_VOCAB_FILES_MAP
							__lowercase      		=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__lowercase      		=       """left"""


							def __init__(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_=False    ,   lowerCAmelCase_=True    ,   lowerCAmelCase_=False    ,   lowerCAmelCase_="<s>"    ,   lowerCAmelCase_="</s>"    ,   lowerCAmelCase_="<unk>"    ,   lowerCAmelCase_="<sep>"    ,   lowerCAmelCase_="<pad>"    ,   lowerCAmelCase_="<cls>"    ,   lowerCAmelCase_="<mask>"    ,   lowerCAmelCase_=["<eop>", "<eod>"]    ,   lowerCAmelCase_ = None    ,   **lowerCAmelCase_    ,   ):





								"""simple docstring"""






								_snake_case   						=		AddedToken(lowerCAmelCase_    ,   lstrip=lowerCAmelCase_    ,   rstrip=lowerCAmelCase_  ) if isinstance(lowerCAmelCase_    ,   lowerCAmelCase_  ) else mask_token

								_snake_case   						=		{} if sp_model_kwargs is None else sp_model_kwargs

								super().__init__(
								    do_lower_case=lowerCAmelCase_    ,   remove_space=lowerCAmelCase_    ,   keep_accents=lowerCAmelCase_    ,   bos_token=lowerCAmelCase_    ,   eos_token=lowerCAmelCase_    ,   unk_token=lowerCAmelCase_    ,   sep_token=lowerCAmelCase_    ,   pad_token=lowerCAmelCase_    ,   cls_token=lowerCAmelCase_    ,   mask_token=lowerCAmelCase_    ,   additional_special_tokens=lowerCAmelCase_    ,   sp_model_kwargs=self.sp_model_kwargs    ,   **lowerCAmelCase_    ,   )

								_snake_case   						=		3

								_snake_case   						=		do_lower_case
								_snake_case   						=		remove_space
								_snake_case   						=		keep_accents
								_snake_case   						=		vocab_file

								_snake_case   						=		spm.SentencePieceProcessor(**self.sp_model_kwargs  )
								self.sp_model.Load(lowerCAmelCase_  )


							@property
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								return len(self.sp_model  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		{self.convert_ids_to_tokens(lowerCAmelCase_  ): i for i in range(self.vocab_size  )}
								vocab.update(self.added_tokens_encoder  )
								return vocab


							def __getstate__(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.__dict__.copy()
								_snake_case   						=		None
								return state


							def __setstate__(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		d

								# for backward compatibility
								if not hasattr(self    ,   'sp_model_kwargs'  ):
									_snake_case   						=		{}

								_snake_case   						=		spm.SentencePieceProcessor(**self.sp_model_kwargs  )
								self.sp_model.Load(self.vocab_file  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								if self.remove_space:
									_snake_case   						=		' '.join(inputs.strip().split()  )
								else:
									_snake_case   						=		inputs
								_snake_case   						=		outputs.replace('``'    ,   '"'  ).replace('\'\''    ,   '"'  )

								if not self.keep_accents:
									_snake_case   						=		unicodedata.normalize('NFKD'    ,   lowerCAmelCase_  )
									_snake_case   						=		''.join([c for c in outputs if not unicodedata.combining(lowerCAmelCase_  )]  )
								if self.do_lower_case:
									_snake_case   						=		outputs.lower()

								return outputs


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		self.preprocess_text(lowerCAmelCase_  )
								_snake_case   						=		self.sp_model.encode(lowerCAmelCase_    ,   out_type=lowerCAmelCase_  )
								_snake_case   						=		[]
								for piece in pieces:
									if len(lowerCAmelCase_  ) > 1 and piece[-1] == str(','  ) and piece[-2].isdigit():
										_snake_case   						=		self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCAmelCase_    ,   ''  )  )
										if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
											if len(cur_pieces[0]  ) == 1:
												_snake_case   						=		cur_pieces[1:]
											else:
												_snake_case   						=		cur_pieces[0][1:]
										cur_pieces.append(piece[-1]  )
										new_pieces.extend(lowerCAmelCase_  )
									else:
										new_pieces.append(lowerCAmelCase_  )

								return new_pieces


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								return self.sp_model.PieceToId(lowerCAmelCase_  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								return self.sp_model.IdToPiece(lowerCAmelCase_  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		''.join(lowerCAmelCase_  ).replace(lowerCAmelCase_    ,   ' '  ).strip()
								return out_string


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_ = False    ,   lowerCAmelCase_ = None    ,   lowerCAmelCase_ = True    ,   **lowerCAmelCase_    ,   ):





								"""simple docstring"""






								_snake_case   						=		kwargs.pop('use_source_tokenizer'    ,   lowerCAmelCase_  )

								_snake_case   						=		self.convert_ids_to_tokens(lowerCAmelCase_    ,   skip_special_tokens=lowerCAmelCase_  )

								# To avoid mixing byte-level and unicode for byte-level BPT
								# we need to build string separately for added tokens and byte-level tokens
								# cf. https://github.com/huggingface/transformers/issues/1133
								_snake_case   						=		[]
								_snake_case   						=		[]
								for token in filtered_tokens:
									if skip_special_tokens and token in self.all_special_ids:
										continue
									if token in self.added_tokens_encoder:
										if current_sub_text:
											sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase_  )  )
											_snake_case   						=		[]
										sub_texts.append(lowerCAmelCase_  )
									else:
										current_sub_text.append(lowerCAmelCase_  )
								if current_sub_text:
									sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase_  )  )

								# Mimic the behavior of the Rust tokenizer:
								# By default, there are no spaces between special tokens
								_snake_case   						=		''.join(lowerCAmelCase_  )

								_snake_case   						=		(
								    clean_up_tokenization_spaces
								    if clean_up_tokenization_spaces is not None
								    else self.clean_up_tokenization_spaces
								)
								if clean_up_tokenization_spaces:
									_snake_case   						=		self.clean_up_tokenization(lowerCAmelCase_  )
									return clean_text
								else:
									return text


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_ = None  ):





								"""simple docstring"""






								_snake_case   						=		[self.sep_token_id]
								_snake_case   						=		[self.cls_token_id]
								if token_ids_a is None:
									return token_ids_a + sep + cls
								return token_ids_a + sep + token_ids_a + sep + cls


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_ = None    ,   lowerCAmelCase_ = False  ):





								"""simple docstring"""






								if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=lowerCAmelCase_    ,   token_ids_a=lowerCAmelCase_    ,   already_has_special_tokens=lowerCAmelCase_  )

								if token_ids_a is not None:
									return ([0] * len(lowerCAmelCase_  )) + [1] + ([0] * len(lowerCAmelCase_  )) + [1, 1]
								return ([0] * len(lowerCAmelCase_  )) + [1, 1]


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_ = None  ):





								"""simple docstring"""






								_snake_case   						=		[self.sep_token_id]
								_snake_case   						=		[2]

								if token_ids_a is None:
									return len(token_ids_a + sep  ) * [0] + cls_segment_id
								return len(token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1] + cls_segment_id




							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_ = None  ):





								"""simple docstring"""






								if not os.path.isdir(lowerCAmelCase_  ):
									logger.error(F'Vocabulary path ({save_directory}) should be a directory'  )
									return
								_snake_case   						=		os.path.join(
								    lowerCAmelCase_    ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )

								if os.path.abspath(self.vocab_file  ) != os.path.abspath(lowerCAmelCase_  ) and os.path.isfile(self.vocab_file  ):
									copyfile(self.vocab_file    ,   lowerCAmelCase_  )
								elif not os.path.isfile(self.vocab_file  ):
									with open(lowerCAmelCase_    ,   'wb'  ) as fi:
										_snake_case   						=		self.sp_model.serialized_model_proto()
										fi.write(lowerCAmelCase_  )

								return (out_vocab_file,)



'''simple docstring'''







import gc
import tempfile
import unittest

import numpy as np
import torch

from diffusers import VersatileDiffusionTextToImagePipeline
from diffusers.utils.testing_utils import nightly, require_torch_gpu, torch_device


lowercase    :							Optional[Any]      =					False


class 						__UpperCAmelCase      (					unittest.TestCase						):
							pass


@nightly
@require_torch_gpu
class 						__UpperCAmelCase      (					unittest.TestCase						):


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								super().tearDown()
								gc.collect()
								torch.cuda.empty_cache()


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		VersatileDiffusionTextToImagePipeline.from_pretrained('shi-labs/versatile-diffusion'  )
								# remove text_unet
								pipe.remove_unused_weights()
								pipe.to(lowerCAmelCase_  )
								pipe.set_progress_bar_config(disable=lowerCAmelCase_  )

								_snake_case   						=		'A painting of a squirrel eating a burger '
								_snake_case   						=		torch.manual_seed(0  )
								_snake_case   						=		pipe(
								    prompt=lowerCAmelCase_    ,   generator=lowerCAmelCase_    ,   guidance_scale=7.5    ,   num_inference_steps=2    ,   output_type='numpy'  ).images

								with tempfile.TemporaryDirectory() as tmpdirname:
									pipe.save_pretrained(lowerCAmelCase_  )
									_snake_case   						=		VersatileDiffusionTextToImagePipeline.from_pretrained(lowerCAmelCase_  )
								pipe.to(lowerCAmelCase_  )
								pipe.set_progress_bar_config(disable=lowerCAmelCase_  )

								_snake_case   						=		generator.manual_seed(0  )
								_snake_case   						=		pipe(
								    prompt=lowerCAmelCase_    ,   generator=lowerCAmelCase_    ,   guidance_scale=7.5    ,   num_inference_steps=2    ,   output_type='numpy'  ).images

								assert np.abs(image - new_image  ).sum() < 1E-5, "Models don't have the same forward pass"




							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		VersatileDiffusionTextToImagePipeline.from_pretrained(
								    'shi-labs/versatile-diffusion'    ,   torch_dtype=torch.floataa  )
								pipe.to(lowerCAmelCase_  )
								pipe.set_progress_bar_config(disable=lowerCAmelCase_  )

								_snake_case   						=		'A painting of a squirrel eating a burger '
								_snake_case   						=		torch.manual_seed(0  )
								_snake_case   						=		pipe(
								    prompt=lowerCAmelCase_    ,   generator=lowerCAmelCase_    ,   guidance_scale=7.5    ,   num_inference_steps=50    ,   output_type='numpy'  ).images

								_snake_case   						=		image[0, 2_53:2_56, 2_53:2_56, -1]

								assert image.shape == (1, 5_12, 5_12, 3)
								_snake_case   						=		np.array([0.3367, 0.3169, 0.2656, 0.3870, 0.4790, 0.3796, 0.4009, 0.4878, 0.4778]  )

								assert np.abs(image_slice.flatten() - expected_slice  ).max() < 1E-2

