




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


lowerCAmelCase_		      = logging.get_logger(__name__)

lowerCAmelCase_		      = {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

lowerCAmelCase_		      = {
    "vocab_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/vocab.txt",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/vocab.txt",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt"
        ),
        "bert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt",
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt"
        ),
        "bert-base-german-dbmdz-cased": "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt",
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json"
        ),
        "bert-base-multilingual-cased": (
            "https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-cased": (
            "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json"
        ),
    },
}

lowerCAmelCase_		      = {
    "bert-base-uncased": 5_12,
    "bert-large-uncased": 5_12,
    "bert-base-cased": 5_12,
    "bert-large-cased": 5_12,
    "bert-base-multilingual-uncased": 5_12,
    "bert-base-multilingual-cased": 5_12,
    "bert-base-chinese": 5_12,
    "bert-base-german-cased": 5_12,
    "bert-large-uncased-whole-word-masking": 5_12,
    "bert-large-cased-whole-word-masking": 5_12,
    "bert-large-uncased-whole-word-masking-finetuned-squad": 5_12,
    "bert-large-cased-whole-word-masking-finetuned-squad": 5_12,
    "bert-base-cased-finetuned-mrpc": 5_12,
    "bert-base-german-dbmdz-cased": 5_12,
    "bert-base-german-dbmdz-uncased": 5_12,
    "TurkuNLP/bert-base-finnish-cased-v1": 5_12,
    "TurkuNLP/bert-base-finnish-uncased-v1": 5_12,
    "wietsedv/bert-base-dutch-cased": 5_12,
}

lowerCAmelCase_		      = {
    "bert-base-uncased": {"do_lower_case": True},
    "bert-large-uncased": {"do_lower_case": True},
    "bert-base-cased": {"do_lower_case": False},
    "bert-large-cased": {"do_lower_case": False},
    "bert-base-multilingual-uncased": {"do_lower_case": True},
    "bert-base-multilingual-cased": {"do_lower_case": False},
    "bert-base-chinese": {"do_lower_case": False},
    "bert-base-german-cased": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking-finetuned-squad": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking-finetuned-squad": {"do_lower_case": False},
    "bert-base-cased-finetuned-mrpc": {"do_lower_case": False},
    "bert-base-german-dbmdz-cased": {"do_lower_case": False},
    "bert-base-german-dbmdz-uncased": {"do_lower_case": True},
    "TurkuNLP/bert-base-finnish-cased-v1": {"do_lower_case": False},
    "TurkuNLP/bert-base-finnish-uncased-v1": {"do_lower_case": True},
    "wietsedv/bert-base-dutch-cased": {"do_lower_case": False},
}
class 						_A					(  _lowerCamelCase      ):
  _UpperCamelCase   :			List[Any] =			VOCAB_FILES_NAMES
  _UpperCamelCase   :			Optional[int] =			PRETRAINED_VOCAB_FILES_MAP
  _UpperCamelCase   :			Tuple =			PRETRAINED_INIT_CONFIGURATION
  _UpperCamelCase   :			Dict =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _UpperCamelCase   :			Tuple =			BertTokenizer






  def __init__(  self		:	Dict      ,						_A		:	int=None      ,						_A		:	Dict=None      ,						_A		:	Union[str, Any]=True      ,						_A		:	Any="[UNK]"      ,						_A		:	Any="[SEP]"      ,						_A		:	Tuple="[PAD]"      ,						_A		:	int="[CLS]"      ,						_A		:	int="[MASK]"      ,						_A		:	List[str]=True      ,						_A		:	Tuple=None      ,						**_A		:	Optional[Any]      ,						)		->    Dict:






       """simple docstring"""







       super().__init__(
           lowerCAmelCase_      ,						tokenizer_file=lowerCAmelCase_      ,						do_lower_case=lowerCAmelCase_      ,						unk_token=lowerCAmelCase_      ,						sep_token=lowerCAmelCase_      ,						pad_token=lowerCAmelCase_      ,						cls_token=lowerCAmelCase_      ,						mask_token=lowerCAmelCase_      ,						tokenize_chinese_chars=lowerCAmelCase_      ,						strip_accents=lowerCAmelCase_      ,						**lowerCAmelCase_      ,						)

       lowercase					:   Any								= json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
       if (
           normalizer_state.get('''lowercase'''      ,						lowerCAmelCase_      ) != do_lower_case
           or normalizer_state.get('''strip_accents'''      ,						lowerCAmelCase_      ) != strip_accents
           or normalizer_state.get('''handle_chinese_chars'''      ,						lowerCAmelCase_      ) != tokenize_chinese_chars
       ):
            lowercase					:   Optional[int]								= getattr(lowerCAmelCase_      ,						normalizer_state.pop('''type'''      )      )
            lowercase					:   Tuple								= do_lower_case
            lowercase					:   List[str]								= strip_accents
            lowercase					:   List[Any]								= tokenize_chinese_chars
            lowercase					:   str								= normalizer_class(**lowerCAmelCase_      )

       lowercase					:   Union[str, Any]								= do_lower_case






  def 					__a			(  self		:	Tuple      ,						_A		:	Union[str, Any]      ,						_A		:	Optional[int]=None      )		->    List[str]:






       """simple docstring"""







       lowercase					:   Any								= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
            output += token_ids_a + [self.sep_token_id]

       return output






  def 					__a			(  self		:	Union[str, Any]      ,						_A		:	Optional[int]      ,						_A		:	str = None      )		->    Any:






       """simple docstring"""







       lowercase					:   Dict								= [self.sep_token_id]
       lowercase					:   Union[str, Any]								= [self.cls_token_id]
       if token_ids_a is None:
            return len(cls + token_ids_a + sep      ) * [0]
       return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]






  def 					__a			(  self		:	List[Any]      ,						_A		:	Dict      ,						_A		:	str = None      )		->    Union[str, Any]:






       """simple docstring"""







       lowercase					:   List[Any]								= self._tokenizer.model.save(lowerCAmelCase_      ,						name=lowerCAmelCase_      )
       return tuple(lowerCAmelCase_      )

'''simple docstring'''







import gc
import tempfile
import unittest

import numpy as np
import torch

from diffusers import VersatileDiffusionTextToImagePipeline
from diffusers.utils.testing_utils import nightly, require_torch_gpu, torch_device


lowercase    :							Optional[Any]      =					False


class 						__UpperCAmelCase      (					unittest.TestCase						):
							pass


@nightly
@require_torch_gpu
class 						__UpperCAmelCase      (					unittest.TestCase						):


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								super().tearDown()
								gc.collect()
								torch.cuda.empty_cache()


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		VersatileDiffusionTextToImagePipeline.from_pretrained('shi-labs/versatile-diffusion'  )
								# remove text_unet
								pipe.remove_unused_weights()
								pipe.to(lowerCAmelCase_  )
								pipe.set_progress_bar_config(disable=lowerCAmelCase_  )

								_snake_case   						=		'A painting of a squirrel eating a burger '
								_snake_case   						=		torch.manual_seed(0  )
								_snake_case   						=		pipe(
								    prompt=lowerCAmelCase_    ,   generator=lowerCAmelCase_    ,   guidance_scale=7.5    ,   num_inference_steps=2    ,   output_type='numpy'  ).images

								with tempfile.TemporaryDirectory() as tmpdirname:
									pipe.save_pretrained(lowerCAmelCase_  )
									_snake_case   						=		VersatileDiffusionTextToImagePipeline.from_pretrained(lowerCAmelCase_  )
								pipe.to(lowerCAmelCase_  )
								pipe.set_progress_bar_config(disable=lowerCAmelCase_  )

								_snake_case   						=		generator.manual_seed(0  )
								_snake_case   						=		pipe(
								    prompt=lowerCAmelCase_    ,   generator=lowerCAmelCase_    ,   guidance_scale=7.5    ,   num_inference_steps=2    ,   output_type='numpy'  ).images

								assert np.abs(image - new_image  ).sum() < 1E-5, "Models don't have the same forward pass"




							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		VersatileDiffusionTextToImagePipeline.from_pretrained(
								    'shi-labs/versatile-diffusion'    ,   torch_dtype=torch.floataa  )
								pipe.to(lowerCAmelCase_  )
								pipe.set_progress_bar_config(disable=lowerCAmelCase_  )

								_snake_case   						=		'A painting of a squirrel eating a burger '
								_snake_case   						=		torch.manual_seed(0  )
								_snake_case   						=		pipe(
								    prompt=lowerCAmelCase_    ,   generator=lowerCAmelCase_    ,   guidance_scale=7.5    ,   num_inference_steps=50    ,   output_type='numpy'  ).images

								_snake_case   						=		image[0, 2_53:2_56, 2_53:2_56, -1]

								assert image.shape == (1, 5_12, 5_12, 3)
								_snake_case   						=		np.array([0.3367, 0.3169, 0.2656, 0.3870, 0.4790, 0.3796, 0.4009, 0.4878, 0.4778]  )

								assert np.abs(image_slice.flatten() - expected_slice  ).max() < 1E-2

