


'''simple docstring'''


import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase__     						=  logging.get_logger(__name__)

lowerCamelCase__     						=  {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

lowerCamelCase__     						=  {
    'vocab_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/vocab.json',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/vocab.json',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/vocab.json',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/merges.txt',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/merges.txt',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/merges.txt',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'
        ),
    },
}

lowerCamelCase__     						=  {
    'roberta-base': 512,
    'roberta-large': 512,
    'roberta-large-mnli': 512,
    'distilroberta-base': 512,
    'roberta-base-openai-detector': 512,
    'roberta-large-openai-detector': 512,
}






@lru_cache()
def 			__lowerCAmelCase			():
		_UpperCAmelCase		:							Any             =   (
		    list(range(ord("!" )   ,			ord("~" ) + 1 ) ) + list(range(ord("¡" )   ,			ord("¬" ) + 1 ) ) + list(range(ord("®" )   ,			ord("ÿ" ) + 1 ) )
		)
		_UpperCAmelCase		:							Dict             =   bs[:]
		_UpperCAmelCase		:							List[Any]             =   0
		for b in range(2**8 ):
				if b not in bs:
						bs.append(__lowerCAmelCase )
						cs.append(2**8 + n )
						n += 1
		_UpperCAmelCase		:							Dict             =   [chr(__lowerCAmelCase ) for n in cs]
		return dict(zip(__lowerCAmelCase   ,			__lowerCAmelCase ) )






def 			__lowerCAmelCase			(__lowerCAmelCase ):
		_UpperCAmelCase		:							Dict             =   set()
		_UpperCAmelCase		:							Dict             =   word[0]
		for char in word[1:]:
				pairs.add((prev_char, char) )
				_UpperCAmelCase		:							List[Any]             =   char
		return pairs






class 						lowerCAmelCase__      (      UpperCAmelCase__						):
			lowerCAmelCase			:						Tuple							 =  VOCAB_FILES_NAMES
			lowerCAmelCase			:						str							 =  PRETRAINED_VOCAB_FILES_MAP
			lowerCAmelCase			:						Optional[int]							 =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCAmelCase			:						List[str]							 =  ["input_ids", "attention_mask"]




			def __init__(		self	:  Tuple						,	lowerCamelCase__	:  Any						,	lowerCamelCase__	:  str						,	lowerCamelCase__	:  List[Any]="replace"						,	lowerCamelCase__	:  int="<s>"						,	lowerCamelCase__	:  int="</s>"						,	lowerCamelCase__	:  Optional[int]="</s>"						,	lowerCamelCase__	:  int="<s>"						,	lowerCamelCase__	:  Any="<unk>"						,	lowerCamelCase__	:  int="<pad>"						,	lowerCamelCase__	:  Union[str, Any]="<mask>"						,	lowerCamelCase__	:  Any=False						,	**lowerCamelCase__	:  Dict						,	) ->Tuple:



					'''simple docstring'''







					_UpperCAmelCase		:							Tuple             =   AddedToken(lowerCamelCase__						,	lstrip=lowerCamelCase__						,	rstrip=lowerCamelCase__			) if isinstance(lowerCamelCase__						,	lowerCamelCase__			) else bos_token
					_UpperCAmelCase		:							Any             =   AddedToken(lowerCamelCase__						,	lstrip=lowerCamelCase__						,	rstrip=lowerCamelCase__			) if isinstance(lowerCamelCase__						,	lowerCamelCase__			) else eos_token
					_UpperCAmelCase		:							List[str]             =   AddedToken(lowerCamelCase__						,	lstrip=lowerCamelCase__						,	rstrip=lowerCamelCase__			) if isinstance(lowerCamelCase__						,	lowerCamelCase__			) else sep_token
					_UpperCAmelCase		:							int             =   AddedToken(lowerCamelCase__						,	lstrip=lowerCamelCase__						,	rstrip=lowerCamelCase__			) if isinstance(lowerCamelCase__						,	lowerCamelCase__			) else cls_token
					_UpperCAmelCase		:							Optional[int]             =   AddedToken(lowerCamelCase__						,	lstrip=lowerCamelCase__						,	rstrip=lowerCamelCase__			) if isinstance(lowerCamelCase__						,	lowerCamelCase__			) else unk_token
					_UpperCAmelCase		:							str             =   AddedToken(lowerCamelCase__						,	lstrip=lowerCamelCase__						,	rstrip=lowerCamelCase__			) if isinstance(lowerCamelCase__						,	lowerCamelCase__			) else pad_token

					# Mask token behave like a normal word, i.e. include the space before it
					_UpperCAmelCase		:							List[str]             =   AddedToken(lowerCamelCase__						,	lstrip=lowerCamelCase__						,	rstrip=lowerCamelCase__			) if isinstance(lowerCamelCase__						,	lowerCamelCase__			) else mask_token

					super().__init__(
					    errors=lowerCamelCase__						,	bos_token=lowerCamelCase__						,	eos_token=lowerCamelCase__						,	unk_token=lowerCamelCase__						,	sep_token=lowerCamelCase__						,	cls_token=lowerCamelCase__						,	pad_token=lowerCamelCase__						,	mask_token=lowerCamelCase__						,	add_prefix_space=lowerCamelCase__						,	**lowerCamelCase__						,	)

					with open(lowerCamelCase__						,	encoding="utf-8"			) as vocab_handle:
							_UpperCAmelCase		:							int             =   json.load(lowerCamelCase__			)
					_UpperCAmelCase		:							List[Any]             =   {v: k for k, v in self.encoder.items()}
					_UpperCAmelCase		:							str             =   errors  # how to handle errors in decoding
					_UpperCAmelCase		:							Optional[Any]             =   bytes_to_unicode()
					_UpperCAmelCase		:							Union[str, Any]             =   {v: k for k, v in self.byte_encoder.items()}
					with open(lowerCamelCase__						,	encoding="utf-8"			) as merges_handle:
							_UpperCAmelCase		:							Optional[Any]             =   merges_handle.read().split("\n"			)[1:-1]
					_UpperCAmelCase		:							int             =   [tuple(merge.split()			) for merge in bpe_merges]
					_UpperCAmelCase		:							Any             =   dict(zip(lowerCamelCase__						,	range(len(lowerCamelCase__			)			)			)			)
					_UpperCAmelCase		:							Dict             =   {}
					_UpperCAmelCase		:							Optional[int]             =   add_prefix_space

					# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
					_UpperCAmelCase		:							int             =   re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"			)




			@property
			def 				lowerCAmelCase__     (		self	:  Optional[int]			) ->int:



					'''simple docstring'''







					return len(self.encoder			)




			def 				lowerCAmelCase__     (		self	:  Union[str, Any]			) ->Optional[int]:



					'''simple docstring'''







					return dict(self.encoder						,	**self.added_tokens_encoder			)




			def 				lowerCAmelCase__     (		self	:  Any						,	lowerCamelCase__	:  Dict			) ->Optional[Any]:



					'''simple docstring'''







					if token in self.cache:
							return self.cache[token]
					_UpperCAmelCase		:							int             =   tuple(lowerCamelCase__			)
					_UpperCAmelCase		:							Optional[int]             =   get_pairs(lowerCamelCase__			)

					if not pairs:
							return token

					while True:
							_UpperCAmelCase		:							Optional[Any]             =   min(lowerCamelCase__						,	key=lambda lowerCamelCase__			: self.bpe_ranks.get(lowerCamelCase__						,	float("inf"			)			)			)
							if bigram not in self.bpe_ranks:
									break
							_UpperCAmelCase       ,     _UpperCAmelCase		:							Any             =   bigram
							_UpperCAmelCase		:							Optional[Any]             =   []
							_UpperCAmelCase		:							Optional[int]             =   0
							while i < len(lowerCamelCase__			):
									try:
											_UpperCAmelCase		:							List[str]             =   word.index(lowerCamelCase__						,	lowerCamelCase__			)
									except ValueError:
											new_word.extend(word[i:]			)
											break
									else:
											new_word.extend(word[i:j]			)
											_UpperCAmelCase		:							str             =   j

									if word[i] == first and i < len(lowerCamelCase__			) - 1 and word[i + 1] == second:
											new_word.append(first + second			)
											i += 2
									else:
											new_word.append(word[i]			)
											i += 1
							_UpperCAmelCase		:							Any             =   tuple(lowerCamelCase__			)
							_UpperCAmelCase		:							List[Any]             =   new_word
							if len(lowerCamelCase__			) == 1:
									break
							else:
									_UpperCAmelCase		:							Optional[Any]             =   get_pairs(lowerCamelCase__			)
					_UpperCAmelCase		:							Dict             =   " ".join(lowerCamelCase__			)
					_UpperCAmelCase		:							int             =   word
					return word




			def 				lowerCAmelCase__     (		self	:  Tuple						,	lowerCamelCase__	:  List[Any]			) ->Any:



					'''simple docstring'''







					_UpperCAmelCase		:							Any             =   []
					for token in re.findall(self.pat						,	lowerCamelCase__			):
							_UpperCAmelCase		:							List[Any]             =   "".join(
							    self.byte_encoder[b] for b in token.encode("utf-8"			)			)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
							bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCamelCase__			).split(" "			)			)
					return bpe_tokens




			def 				lowerCAmelCase__     (		self	:  str						,	lowerCamelCase__	:  Any			) ->Dict:



					'''simple docstring'''







					return self.encoder.get(lowerCamelCase__						,	self.encoder.get(self.unk_token			)			)




			def 				lowerCAmelCase__     (		self	:  Any						,	lowerCamelCase__	:  Optional[int]			) ->Dict:



					'''simple docstring'''







					return self.decoder.get(lowerCamelCase__			)




			def 				lowerCAmelCase__     (		self	:  Optional[Any]						,	lowerCamelCase__	:  List[str]			) ->Tuple:



					'''simple docstring'''







					_UpperCAmelCase		:							Union[str, Any]             =   "".join(lowerCamelCase__			)
					_UpperCAmelCase		:							Tuple             =   bytearray([self.byte_decoder[c] for c in text]			).decode("utf-8"						,	errors=self.errors			)
					return text




			def 				lowerCAmelCase__     (		self	:  Any						,	lowerCamelCase__	:  str						,	lowerCamelCase__	:  Optional[str] = None			) ->Tuple[str]:



					'''simple docstring'''







					if not os.path.isdir(lowerCamelCase__			):
							logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""			)
							return
					_UpperCAmelCase		:							List[str]             =   os.path.join(
					    lowerCamelCase__						,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]			)
					_UpperCAmelCase		:							int             =   os.path.join(
					    lowerCamelCase__						,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]			)

					with open(lowerCamelCase__						,	"w"						,	encoding="utf-8"			) as f:
							f.write(json.dumps(self.encoder						,	indent=2						,	sort_keys=lowerCamelCase__						,	ensure_ascii=lowerCamelCase__			) + "\n"			)

					_UpperCAmelCase		:							Optional[int]             =   0
					with open(lowerCamelCase__						,	"w"						,	encoding="utf-8"			) as writer:
							writer.write("#version: 0.2\n"			)
							for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,	key=lambda lowerCamelCase__			: kv[1]			):
									if index != token_index:
											logger.warning(
											    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
											    " Please check that the tokenizer is not corrupted!"			)
											_UpperCAmelCase		:							Union[str, Any]             =   token_index
									writer.write(" ".join(lowerCamelCase__			) + "\n"			)
									index += 1

					return vocab_file, merge_file




			def 				lowerCAmelCase__     (		self	:  Tuple						,	lowerCamelCase__	:  List[int]						,	lowerCamelCase__	:  Optional[List[int]] = None			) ->List[int]:



					'''simple docstring'''







					if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
					_UpperCAmelCase		:							Any             =   [self.cls_token_id]
					_UpperCAmelCase		:							List[Any]             =   [self.sep_token_id]
					return cls + token_ids_a + sep + sep + token_ids_a + sep




			def 				lowerCAmelCase__     (		self	:  List[Any]						,	lowerCamelCase__	:  List[int]						,	lowerCamelCase__	:  Optional[List[int]] = None						,	lowerCamelCase__	:  bool = False			) ->List[int]:



					'''simple docstring'''







					if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=lowerCamelCase__						,	token_ids_a=lowerCamelCase__						,	already_has_special_tokens=lowerCamelCase__			)

					if token_ids_a is None:
							return [1] + ([0] * len(lowerCamelCase__			)) + [1]
					return [1] + ([0] * len(lowerCamelCase__			)) + [1, 1] + ([0] * len(lowerCamelCase__			)) + [1]




			def 				lowerCAmelCase__     (		self	:  Any						,	lowerCamelCase__	:  List[int]						,	lowerCamelCase__	:  Optional[List[int]] = None			) ->List[int]:



					'''simple docstring'''







					_UpperCAmelCase		:							List[Any]             =   [self.sep_token_id]
					_UpperCAmelCase		:							Union[str, Any]             =   [self.cls_token_id]

					if token_ids_a is None:
							return len(cls + token_ids_a + sep			) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]






			def 				lowerCAmelCase__     (		self	:  Optional[int]						,	lowerCamelCase__	:  List[str]						,	lowerCamelCase__	:  int=False						,	**lowerCamelCase__	:  Dict			) ->List[str]:



					'''simple docstring'''







					_UpperCAmelCase		:							List[str]             =   kwargs.pop("add_prefix_space"						,	self.add_prefix_space			)
					if (is_split_into_words or add_prefix_space) and (len(lowerCamelCase__			) > 0 and not text[0].isspace()):
							_UpperCAmelCase		:							Union[str, Any]             =   " " + text
					return (text, kwargs)







'''simple docstring'''


from ..utils import deprecate
from .attention_processor import (  # noqa: F401
    Attention,
    AttentionProcessor,
    AttnAddedKVProcessor,
    AttnProcessor2_0,
    LoRAAttnProcessor,
    LoRALinearLayer,
    LoRAXFormersAttnProcessor,
    SlicedAttnAddedKVProcessor,
    SlicedAttnProcessor,
    XFormersAttnProcessor,
)
from .attention_processor import AttnProcessor as AttnProcessorRename  # noqa: F401


deprecate(
    'cross_attention',
    '0.20.0',
    'Importing from cross_attention is deprecated. Please import from diffusers.models.attention_processor instead.',
    standard_warn=False,
)


lowerCamelCase__     						=  AttentionProcessor






class 						lowerCAmelCase__      (      UpperCAmelCase__						):




			def __init__(		self	:  Union[str, Any]						,	*lowerCamelCase__	:  int						,	**lowerCamelCase__	:  List[str]			) ->Any:



					'''simple docstring'''







					_UpperCAmelCase		:							Dict             =   F"""{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {''.join(self.__class__.__name__.split('Cross'			)			)} instead."""
					deprecate("cross_attention"						,	"0.20.0"						,	lowerCamelCase__						,	standard_warn=lowerCamelCase__			)
					super().__init__(*lowerCamelCase__						,	**lowerCamelCase__			)






class 						lowerCAmelCase__      (      UpperCAmelCase__						):




			def __init__(		self	:  Optional[Any]						,	*lowerCamelCase__	:  Any						,	**lowerCamelCase__	:  Tuple			) ->List[Any]:



					'''simple docstring'''







					_UpperCAmelCase		:							int             =   F"""{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {''.join(self.__class__.__name__.split('Cross'			)			)} instead."""
					deprecate("cross_attention"						,	"0.20.0"						,	lowerCamelCase__						,	standard_warn=lowerCamelCase__			)
					super().__init__(*lowerCamelCase__						,	**lowerCamelCase__			)






class 						lowerCAmelCase__      (      UpperCAmelCase__						):




			def __init__(		self	:  List[Any]						,	*lowerCamelCase__	:  List[Any]						,	**lowerCamelCase__	:  List[str]			) ->List[Any]:



					'''simple docstring'''







					_UpperCAmelCase		:							List[str]             =   F"""{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {''.join(self.__class__.__name__.split('Cross'			)			)} instead."""
					deprecate("cross_attention"						,	"0.20.0"						,	lowerCamelCase__						,	standard_warn=lowerCamelCase__			)
					super().__init__(*lowerCamelCase__						,	**lowerCamelCase__			)






class 						lowerCAmelCase__      (      UpperCAmelCase__						):




			def __init__(		self	:  int						,	*lowerCamelCase__	:  Tuple						,	**lowerCamelCase__	:  Union[str, Any]			) ->int:



					'''simple docstring'''







					_UpperCAmelCase		:							Dict             =   F"""{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {''.join(self.__class__.__name__.split('Cross'			)			)} instead."""
					deprecate("cross_attention"						,	"0.20.0"						,	lowerCamelCase__						,	standard_warn=lowerCamelCase__			)
					super().__init__(*lowerCamelCase__						,	**lowerCamelCase__			)






class 						lowerCAmelCase__      (      UpperCAmelCase__						):




			def __init__(		self	:  int						,	*lowerCamelCase__	:  List[str]						,	**lowerCamelCase__	:  Tuple			) ->Union[str, Any]:



					'''simple docstring'''







					_UpperCAmelCase		:							Optional[Any]             =   F"""{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {''.join(self.__class__.__name__.split('Cross'			)			)} instead."""
					deprecate("cross_attention"						,	"0.20.0"						,	lowerCamelCase__						,	standard_warn=lowerCamelCase__			)
					super().__init__(*lowerCamelCase__						,	**lowerCamelCase__			)






class 						lowerCAmelCase__      (      UpperCAmelCase__						):




			def __init__(		self	:  List[Any]						,	*lowerCamelCase__	:  int						,	**lowerCamelCase__	:  Optional[int]			) ->Optional[int]:



					'''simple docstring'''







					_UpperCAmelCase		:							List[Any]             =   F"""{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {''.join(self.__class__.__name__.split('Cross'			)			)} instead."""
					deprecate("cross_attention"						,	"0.20.0"						,	lowerCamelCase__						,	standard_warn=lowerCamelCase__			)
					super().__init__(*lowerCamelCase__						,	**lowerCamelCase__			)






class 						lowerCAmelCase__      (      UpperCAmelCase__						):




			def __init__(		self	:  Any						,	*lowerCamelCase__	:  Optional[Any]						,	**lowerCamelCase__	:  Union[str, Any]			) ->List[str]:



					'''simple docstring'''







					_UpperCAmelCase		:							int             =   F"""{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {''.join(self.__class__.__name__.split('Cross'			)			)} instead."""
					deprecate("cross_attention"						,	"0.20.0"						,	lowerCamelCase__						,	standard_warn=lowerCamelCase__			)
					super().__init__(*lowerCamelCase__						,	**lowerCamelCase__			)






class 						lowerCAmelCase__      (      UpperCAmelCase__						):




			def __init__(		self	:  Optional[Any]						,	*lowerCamelCase__	:  List[Any]						,	**lowerCamelCase__	:  Dict			) ->Any:



					'''simple docstring'''







					_UpperCAmelCase		:							int             =   F"""{self.__class__.__name__} is deprecated and will be removed in `0.20.0`. Please use `from diffusers.models.attention_processor import {''.join(self.__class__.__name__.split('Cross'			)			)} instead."""
					deprecate("cross_attention"						,	"0.20.0"						,	lowerCamelCase__						,	standard_warn=lowerCamelCase__			)
					super().__init__(*lowerCamelCase__						,	**lowerCamelCase__			)



