


'''simple docstring'''

import gc
import random
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, DDPMScheduler, StableDiffusionUpscalePipeline, UNetaDConditionModel
from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu


enable_full_determinism()





class      snake_case    (    unittest.TestCase			):





  """simple docstring"""




  def   snake_case		(    self     ):







   """simple docstring"""
   # clean up the VRAM after each test
   super().tearDown()
   gc.collect()
   torch.cuda.empty_cache()






  @property
  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  1
   lowerCamelCase_      =  3
   lowerCamelCase_      =  (32, 32)

   lowerCamelCase_      =  floats_tensor((batch_size, num_channels) + sizes			,  rng=random.Random(0     )     ).to(UpperCamelCase     )
   return image






  @property
  def   snake_case		(    self     ):







   """simple docstring"""
   torch.manual_seed(0     )
   lowerCamelCase_      =  UNetaDConditionModel(
       block_out_channels=(32, 32, 64)			,  layers_per_block=2			,  sample_size=32			,  in_channels=7			,  out_channels=4			,  down_block_types=("DownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D")			,  up_block_types=("CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "UpBlock2D")			,  cross_attention_dim=32			,  attention_head_dim=8			,  use_linear_projection=UpperCamelCase			,  only_cross_attention=(True, True, False)			,  num_class_embeds=100			,  )
   return model






  @property
  def   snake_case		(    self     ):







   """simple docstring"""
   torch.manual_seed(0     )
   lowerCamelCase_      =  AutoencoderKL(
       block_out_channels=[32, 32, 64]			,  in_channels=3			,  out_channels=3			,  down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D", "DownEncoderBlock2D"]			,  up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D"]			,  latent_channels=4			,  )
   return model






  @property
  def   snake_case		(    self     ):







   """simple docstring"""
   torch.manual_seed(0     )
   lowerCamelCase_      =  CLIPTextConfig(
       bos_token_id=0			,  eos_token_id=2			,  hidden_size=32			,  intermediate_size=37			,  layer_norm_eps=1e-05			,  num_attention_heads=4			,  num_hidden_layers=5			,  pad_token_id=1			,  vocab_size=1000			,  hidden_act="gelu"			,  projection_dim=512			,  )
   return CLIPTextModel(UpperCamelCase     )






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  "cpu"  # ensure determinism for the device-dependent torch.Generator
   lowerCamelCase_      =  self.dummy_cond_unet_upscale
   lowerCamelCase_      =  DDPMScheduler()
   lowerCamelCase_      =  DDIMScheduler(prediction_type="v_prediction"     )
   lowerCamelCase_      =  self.dummy_vae
   lowerCamelCase_      =  self.dummy_text_encoder
   lowerCamelCase_      =  CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"     )

   lowerCamelCase_      =  self.dummy_image.cpu().permute(0			,  2			,  3			,  1     )[0]
   lowerCamelCase_      =  Image.fromarray(np.uinta(UpperCamelCase     )     ).convert("RGB"     ).resize((64, 64)     )

   # make sure here that pndm scheduler skips prk
   lowerCamelCase_      =  StableDiffusionUpscalePipeline(
       unet=UpperCamelCase			,  low_res_scheduler=UpperCamelCase			,  scheduler=UpperCamelCase			,  vae=UpperCamelCase			,  text_encoder=UpperCamelCase			,  tokenizer=UpperCamelCase			,  max_noise_level=350			,  )
   lowerCamelCase_      =  sd_pipe.to(UpperCamelCase     )
   sd_pipe.set_progress_bar_config(disable=UpperCamelCase     )

   lowerCamelCase_      =  "A painting of a squirrel eating a burger"
   lowerCamelCase_      =  torch.Generator(device=UpperCamelCase     ).manual_seed(0     )
   lowerCamelCase_      =  sd_pipe(
       [prompt]			,  image=UpperCamelCase			,  generator=UpperCamelCase			,  guidance_scale=6.0			,  noise_level=20			,  num_inference_steps=2			,  output_type="np"			,  )

   lowerCamelCase_      =  output.images

   lowerCamelCase_      =  torch.Generator(device=UpperCamelCase     ).manual_seed(0     )
   lowerCamelCase_      =  sd_pipe(
       [prompt]			,  image=UpperCamelCase			,  generator=UpperCamelCase			,  guidance_scale=6.0			,  noise_level=20			,  num_inference_steps=2			,  output_type="np"			,  return_dict=UpperCamelCase			,  )[0]

   lowerCamelCase_      =  image[0, -3:, -3:, -1]
   lowerCamelCase_      =  image_from_tuple[0, -3:, -3:, -1]

   lowerCamelCase_      =  low_res_image.size[0] * 4
   assert image.shape == (1, expected_height_width, expected_height_width, 3)
   lowerCamelCase_      =  np.array([0.3_113, 0.3_910, 0.4_272, 0.4_859, 0.5_061, 0.4_652, 0.5_362, 0.5_715, 0.5_661]     )

   assert np.abs(image_slice.flatten() - expected_slice     ).max() < 1e-2
   assert np.abs(image_from_tuple_slice.flatten() - expected_slice     ).max() < 1e-2






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  "cpu"  # ensure determinism for the device-dependent torch.Generator
   lowerCamelCase_      =  self.dummy_cond_unet_upscale
   lowerCamelCase_      =  DDPMScheduler()
   lowerCamelCase_      =  DDIMScheduler(prediction_type="v_prediction"     )
   lowerCamelCase_      =  self.dummy_vae
   lowerCamelCase_      =  self.dummy_text_encoder
   lowerCamelCase_      =  CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"     )

   lowerCamelCase_      =  self.dummy_image.cpu().permute(0			,  2			,  3			,  1     )[0]
   lowerCamelCase_      =  Image.fromarray(np.uinta(UpperCamelCase     )     ).convert("RGB"     ).resize((64, 64)     )

   # make sure here that pndm scheduler skips prk
   lowerCamelCase_      =  StableDiffusionUpscalePipeline(
       unet=UpperCamelCase			,  low_res_scheduler=UpperCamelCase			,  scheduler=UpperCamelCase			,  vae=UpperCamelCase			,  text_encoder=UpperCamelCase			,  tokenizer=UpperCamelCase			,  max_noise_level=350			,  )
   lowerCamelCase_      =  sd_pipe.to(UpperCamelCase     )
   sd_pipe.set_progress_bar_config(disable=UpperCamelCase     )

   lowerCamelCase_      =  "A painting of a squirrel eating a burger"
   lowerCamelCase_      =  sd_pipe(
       2 * [prompt]			,  image=2 * [low_res_image]			,  guidance_scale=6.0			,  noise_level=20			,  num_inference_steps=2			,  output_type="np"			,  )
   lowerCamelCase_      =  output.images
   assert image.shape[0] == 2

   lowerCamelCase_      =  torch.Generator(device=UpperCamelCase     ).manual_seed(0     )
   lowerCamelCase_      =  sd_pipe(
       [prompt]			,  image=UpperCamelCase			,  generator=UpperCamelCase			,  num_images_per_prompt=2			,  guidance_scale=6.0			,  noise_level=20			,  num_inference_steps=2			,  output_type="np"			,  )
   lowerCamelCase_      =  output.images
   assert image.shape[0] == 2







  @unittest.skipIf(torch_device != "cuda"			,  "This test requires a GPU"     )
  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  self.dummy_cond_unet_upscale
   lowerCamelCase_      =  DDPMScheduler()
   lowerCamelCase_      =  DDIMScheduler(prediction_type="v_prediction"     )
   lowerCamelCase_      =  self.dummy_vae
   lowerCamelCase_      =  self.dummy_text_encoder
   lowerCamelCase_      =  CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"     )

   lowerCamelCase_      =  self.dummy_image.cpu().permute(0			,  2			,  3			,  1     )[0]
   lowerCamelCase_      =  Image.fromarray(np.uinta(UpperCamelCase     )     ).convert("RGB"     ).resize((64, 64)     )

   # put models in fp16, except vae as it overflows in fp16
   lowerCamelCase_      =  unet.half()
   lowerCamelCase_      =  text_encoder.half()

   # make sure here that pndm scheduler skips prk
   lowerCamelCase_      =  StableDiffusionUpscalePipeline(
       unet=UpperCamelCase			,  low_res_scheduler=UpperCamelCase			,  scheduler=UpperCamelCase			,  vae=UpperCamelCase			,  text_encoder=UpperCamelCase			,  tokenizer=UpperCamelCase			,  max_noise_level=350			,  )
   lowerCamelCase_      =  sd_pipe.to(UpperCamelCase     )
   sd_pipe.set_progress_bar_config(disable=UpperCamelCase     )

   lowerCamelCase_      =  "A painting of a squirrel eating a burger"
   lowerCamelCase_      =  torch.manual_seed(0     )
   lowerCamelCase_      =  sd_pipe(
       [prompt]			,  image=UpperCamelCase			,  generator=UpperCamelCase			,  num_inference_steps=2			,  output_type="np"			,  ).images

   lowerCamelCase_      =  low_res_image.size[0] * 4
   assert image.shape == (1, expected_height_width, expected_height_width, 3)





@slow
@require_torch_gpu
class      snake_case    (    unittest.TestCase			):





  """simple docstring"""




  def   snake_case		(    self     ):







   """simple docstring"""
   # clean up the VRAM after each test
   super().tearDown()
   gc.collect()
   torch.cuda.empty_cache()






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  load_image(
       "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"
       "/sd2-upscale/low_res_cat.png"     )
   lowerCamelCase_      =  load_numpy(
       "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-upscale"
       "/upsampled_cat.npy"     )

   lowerCamelCase_      =  "stabilityai/stable-diffusion-x4-upscaler"
   lowerCamelCase_      =  StableDiffusionUpscalePipeline.from_pretrained(UpperCamelCase     )
   pipe.to(UpperCamelCase     )
   pipe.set_progress_bar_config(disable=UpperCamelCase     )
   pipe.enable_attention_slicing()

   lowerCamelCase_      =  "a cat sitting on a park bench"

   lowerCamelCase_      =  torch.manual_seed(0     )
   lowerCamelCase_      =  pipe(
       prompt=UpperCamelCase			,  image=UpperCamelCase			,  generator=UpperCamelCase			,  output_type="np"			,  )
   lowerCamelCase_      =  output.images[0]

   assert image.shape == (512, 512, 3)
   assert np.abs(expected_image - image     ).max() < 1e-3






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  load_image(
       "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"
       "/sd2-upscale/low_res_cat.png"     )
   lowerCamelCase_      =  load_numpy(
       "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-upscale"
       "/upsampled_cat_fp16.npy"     )

   lowerCamelCase_      =  "stabilityai/stable-diffusion-x4-upscaler"
   lowerCamelCase_      =  StableDiffusionUpscalePipeline.from_pretrained(
       UpperCamelCase			,  torch_dtype=torch.floataa			,  )
   pipe.to(UpperCamelCase     )
   pipe.set_progress_bar_config(disable=UpperCamelCase     )
   pipe.enable_attention_slicing()

   lowerCamelCase_      =  "a cat sitting on a park bench"

   lowerCamelCase_      =  torch.manual_seed(0     )
   lowerCamelCase_      =  pipe(
       prompt=UpperCamelCase			,  image=UpperCamelCase			,  generator=UpperCamelCase			,  output_type="np"			,  )
   lowerCamelCase_      =  output.images[0]

   assert image.shape == (512, 512, 3)
   assert np.abs(expected_image - image     ).max() < 5e-1







  def   snake_case		(    self     ):







   """simple docstring"""
   torch.cuda.empty_cache()
   torch.cuda.reset_max_memory_allocated()
   torch.cuda.reset_peak_memory_stats()

   lowerCamelCase_      =  load_image(
       "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"
       "/sd2-upscale/low_res_cat.png"     )

   lowerCamelCase_      =  "stabilityai/stable-diffusion-x4-upscaler"
   lowerCamelCase_      =  StableDiffusionUpscalePipeline.from_pretrained(
       UpperCamelCase			,  torch_dtype=torch.floataa			,  )
   pipe.to(UpperCamelCase     )
   pipe.set_progress_bar_config(disable=UpperCamelCase     )
   pipe.enable_attention_slicing(1     )
   pipe.enable_sequential_cpu_offload()

   lowerCamelCase_      =  "a cat sitting on a park bench"

   lowerCamelCase_      =  torch.manual_seed(0     )
   lowerCamelCase_      =  pipe(
       prompt=UpperCamelCase			,  image=UpperCamelCase			,  generator=UpperCamelCase			,  num_inference_steps=5			,  output_type="np"			,  )

   lowerCamelCase_      =  torch.cuda.max_memory_allocated()
   # make sure that less than 2.9 GB is allocated
   assert mem_bytes < 2.9 * 10**9









import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


SCREAMING_SNAKE_CASE__       :      Optional[Any]  =   logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__       :      Tuple  =   {'vocab_file': 'spiece.model'}

SCREAMING_SNAKE_CASE__       :      int  =   {
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    }
}

SCREAMING_SNAKE_CASE__       :      str  =   {
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

# Segments (not really needed)
SCREAMING_SNAKE_CASE__       :      Dict  =   0
SCREAMING_SNAKE_CASE__       :      Tuple  =   1
SCREAMING_SNAKE_CASE__       :      Optional[int]  =   2
SCREAMING_SNAKE_CASE__       :      List[str]  =   3
SCREAMING_SNAKE_CASE__       :      Optional[int]  =   4




class 		UpperCamelCase__	(lowerCAmelCase__     ):

    '''simple docstring'''



    lowerCamelCase_      :	Dict	     =      VOCAB_FILES_NAMES
    lowerCamelCase_      :	Optional[Any]	     =      PRETRAINED_VOCAB_FILES_MAP
    lowerCamelCase_      :	Optional[int]	     =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCamelCase_      :	List[str]	     =      """left"""







    def __init__(    self     ,  UpperCamelCase__     ,  UpperCamelCase__=False     ,  UpperCamelCase__=True     ,  UpperCamelCase__=False     ,  UpperCamelCase__="<s>"     ,  UpperCamelCase__="</s>"     ,  UpperCamelCase__="<unk>"     ,  UpperCamelCase__="<sep>"     ,  UpperCamelCase__="<pad>"     ,  UpperCamelCase__="<cls>"     ,  UpperCamelCase__="<mask>"     ,  UpperCamelCase__=["<eop>", "<eod>"]     ,  UpperCamelCase__ = None     ,  **UpperCamelCase__     ,  )     ->      None:
           # Mask token behave like a normal word, i.e. include the space before it
           lowerCamelCase   :      str			 =					AddedToken(UpperCamelCase__     ,  lstrip=UpperCamelCase__     ,  rstrip=UpperCamelCase__     ) if isinstance(UpperCamelCase__     ,  UpperCamelCase__     ) else mask_token

           lowerCamelCase   :      Dict			 =					{} if sp_model_kwargs is None else sp_model_kwargs

           super().__init__(
               do_lower_case=UpperCamelCase__     ,  remove_space=UpperCamelCase__     ,  keep_accents=UpperCamelCase__     ,  bos_token=UpperCamelCase__     ,  eos_token=UpperCamelCase__     ,  unk_token=UpperCamelCase__     ,  sep_token=UpperCamelCase__     ,  pad_token=UpperCamelCase__     ,  cls_token=UpperCamelCase__     ,  mask_token=UpperCamelCase__     ,  additional_special_tokens=UpperCamelCase__     ,  sp_model_kwargs=self.sp_model_kwargs     ,  **UpperCamelCase__     ,  )

           lowerCamelCase   :      Any			 =					3

           lowerCamelCase   :      Optional[Any]			 =					do_lower_case
           lowerCamelCase   :      List[Any]			 =					remove_space
           lowerCamelCase   :      str			 =					keep_accents
           lowerCamelCase   :      List[Any]			 =					vocab_file

           lowerCamelCase   :      int			 =					spm.SentencePieceProcessor(**self.sp_model_kwargs     )
           self.sp_model.Load(UpperCamelCase__     )







    @property
    def     _lowercase     (    self     )     ->      Optional[Any]:
           return len(self.sp_model     )







    def     _lowercase     (    self     )     ->      Optional[int]:
           lowerCamelCase   :      int			 =					{self.convert_ids_to_tokens(UpperCamelCase__     ): i for i in range(self.vocab_size     )}
           vocab.update(self.added_tokens_encoder     )
           return vocab







    def __getstate__(    self     )     ->      Optional[Any]:
           lowerCamelCase   :      Optional[int]			 =					self.__dict__.copy()
           lowerCamelCase   :      Union[str, Any]			 =					None
           return state







    def __setstate__(    self     ,  UpperCamelCase__     )     ->      int:
           lowerCamelCase   :      int			 =					d

           # for backward compatibility
           if not hasattr(self     ,  "sp_model_kwargs"     ):
                  lowerCamelCase   :      Any			 =					{}

           lowerCamelCase   :      Optional[int]			 =					spm.SentencePieceProcessor(**self.sp_model_kwargs     )
           self.sp_model.Load(self.vocab_file     )







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      Any:
           if self.remove_space:
                  lowerCamelCase   :      Dict			 =					" ".join(inputs.strip().split()     )
           else:
                  lowerCamelCase   :      Union[str, Any]			 =					inputs
           lowerCamelCase   :      Optional[Any]			 =					outputs.replace("``"     ,  "\""     ).replace("''"     ,  "\""     )

           if not self.keep_accents:
                  lowerCamelCase   :      Optional[int]			 =					unicodedata.normalize("NFKD"     ,  UpperCamelCase__     )
                  lowerCamelCase   :      List[Any]			 =					"".join([c for c in outputs if not unicodedata.combining(UpperCamelCase__     )]     )
           if self.do_lower_case:
                  lowerCamelCase   :      List[str]			 =					outputs.lower()

           return outputs







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      List[str]:
           lowerCamelCase   :      Optional[Any]			 =					self.preprocess_text(UpperCamelCase__     )
           lowerCamelCase   :      Dict			 =					self.sp_model.encode(UpperCamelCase__     ,  out_type=UpperCamelCase__     )
           lowerCamelCase   :      Dict			 =					[]
           for piece in pieces:
                  if len(UpperCamelCase__     ) > 1 and piece[-1] == str(","     ) and piece[-2].isdigit():
                         lowerCamelCase   :      List[Any]			 =					self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCamelCase__     ,  ""     )     )
                         if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                                if len(cur_pieces[0]     ) == 1:
                                       lowerCamelCase   :      Union[str, Any]			 =					cur_pieces[1:]
                                else:
                                       lowerCamelCase   :      Optional[int]			 =					cur_pieces[0][1:]
                         cur_pieces.append(piece[-1]     )
                         new_pieces.extend(UpperCamelCase__     )
                  else:
                         new_pieces.append(UpperCamelCase__     )

           return new_pieces







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      int:
           return self.sp_model.PieceToId(UpperCamelCase__     )







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      Tuple:
           return self.sp_model.IdToPiece(UpperCamelCase__     )







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      List[str]:
           lowerCamelCase   :      Union[str, Any]			 =					"".join(UpperCamelCase__     ).replace(UpperCamelCase__     ,  " "     ).strip()
           return out_string







    def     _lowercase     (    self     ,  UpperCamelCase__     ,  UpperCamelCase__ = False     ,  UpperCamelCase__ = None     ,  UpperCamelCase__ = True     ,  **UpperCamelCase__     ,  )     ->      str:
           lowerCamelCase   :      Optional[int]			 =					kwargs.pop("use_source_tokenizer"     ,  UpperCamelCase__     )

           lowerCamelCase   :      Optional[int]			 =					self.convert_ids_to_tokens(UpperCamelCase__     ,  skip_special_tokens=UpperCamelCase__     )

           # To avoid mixing byte-level and unicode for byte-level BPT
           # we need to build string separately for added tokens and byte-level tokens
           # cf. https://github.com/huggingface/transformers/issues/1133
           lowerCamelCase   :      Any			 =					[]
           lowerCamelCase   :      Any			 =					[]
           for token in filtered_tokens:
                  if skip_special_tokens and token in self.all_special_ids:
                         continue
                  if token in self.added_tokens_encoder:
                         if current_sub_text:
                                sub_texts.append(self.convert_tokens_to_string(UpperCamelCase__     )     )
                                lowerCamelCase   :      int			 =					[]
                         sub_texts.append(UpperCamelCase__     )
                  else:
                         current_sub_text.append(UpperCamelCase__     )
           if current_sub_text:
                  sub_texts.append(self.convert_tokens_to_string(UpperCamelCase__     )     )

           # Mimic the behavior of the Rust tokenizer:
           # By default, there are no spaces between special tokens
           lowerCamelCase   :      Union[str, Any]			 =					"".join(UpperCamelCase__     )

           lowerCamelCase   :      Tuple			 =					(
               clean_up_tokenization_spaces
               if clean_up_tokenization_spaces is not None
               else self.clean_up_tokenization_spaces
           )
           if clean_up_tokenization_spaces:
                  lowerCamelCase   :      int			 =					self.clean_up_tokenization(UpperCamelCase__     )
                  return clean_text
           else:
                  return text







    def     _lowercase     (    self     ,  UpperCamelCase__     ,  UpperCamelCase__ = None     )     ->      List[int]:
           lowerCamelCase   :      str			 =					[self.sep_token_id]
           lowerCamelCase   :      Optional[int]			 =					[self.cls_token_id]
           if token_ids_a is None:
                  return token_ids_a + sep + cls
           return token_ids_a + sep + token_ids_a + sep + cls







    def     _lowercase     (    self     ,  UpperCamelCase__     ,  UpperCamelCase__ = None     ,  UpperCamelCase__ = False     )     ->      List[int]:

           if already_has_special_tokens:
                  return super().get_special_tokens_mask(
                      token_ids_a=UpperCamelCase__     ,  token_ids_a=UpperCamelCase__     ,  already_has_special_tokens=UpperCamelCase__     )

           if token_ids_a is not None:
                  return ([0] * len(UpperCamelCase__     )) + [1] + ([0] * len(UpperCamelCase__     )) + [1, 1]
           return ([0] * len(UpperCamelCase__     )) + [1, 1]







    def     _lowercase     (    self     ,  UpperCamelCase__     ,  UpperCamelCase__ = None     )     ->      List[int]:
           lowerCamelCase   :      Any			 =					[self.sep_token_id]
           lowerCamelCase   :      List[str]			 =					[2]

           if token_ids_a is None:
                  return len(token_ids_a + sep     ) * [0] + cls_segment_id
           return len(token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1] + cls_segment_id







    def     _lowercase     (    self     ,  UpperCamelCase__     ,  UpperCamelCase__ = None     )     ->      Tuple[str]:
           if not os.path.isdir(UpperCamelCase__     ):
                  logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''     )
                  return
           lowerCamelCase   :      Union[str, Any]			 =					os.path.join(
               UpperCamelCase__     ,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )

           if os.path.abspath(self.vocab_file     ) != os.path.abspath(UpperCamelCase__     ) and os.path.isfile(self.vocab_file     ):
                  copyfile(self.vocab_file     ,  UpperCamelCase__     )
           elif not os.path.isfile(self.vocab_file     ):
                  with open(UpperCamelCase__     ,  "wb"     ) as fi:
                         lowerCamelCase   :      str			 =					self.sp_model.serialized_model_proto()
                         fi.write(UpperCamelCase__     )

           return (out_vocab_file,)






