




'''simple docstring'''







import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
						from transformers.pipelines.conversational import Conversation

_lowerCamelCase				:		int				      =    logging.get_logger(__name__)

_lowerCamelCase				:		Any				      =    {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

_lowerCamelCase				:		List[Any]				      =    {
    'vocab_file': {
        'microsoft/deberta-base': 'https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json',
        'microsoft/deberta-large': 'https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json',
        'microsoft/deberta-xlarge': 'https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json',
        'microsoft/deberta-base-mnli': 'https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json',
        'microsoft/deberta-large-mnli': 'https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json',
        'microsoft/deberta-xlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'microsoft/deberta-base': 'https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt',
        'microsoft/deberta-large': 'https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt',
        'microsoft/deberta-xlarge': 'https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt',
        'microsoft/deberta-base-mnli': 'https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt',
        'microsoft/deberta-large-mnli': 'https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt',
        'microsoft/deberta-xlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt'
        ),
    },
}

_lowerCamelCase				:		Optional[Any]				      =    {
    'microsoft/deberta-base': 512,
    'microsoft/deberta-large': 512,
    'microsoft/deberta-xlarge': 512,
    'microsoft/deberta-base-mnli': 512,
    'microsoft/deberta-large-mnli': 512,
    'microsoft/deberta-xlarge-mnli': 512,
}

_lowerCamelCase				:		Dict				      =    {
    'microsoft/deberta-base': {'do_lower_case': False},
    'microsoft/deberta-large': {'do_lower_case': False},
}






def 						_lowercase						(      ):



		'''simple docstring'''







		_snake_case      			=			(
		    list(range(ord("!"    )		,				ord("~"    ) + 1    )    ) + list(range(ord("¡"    )		,				ord("¬"    ) + 1    )    ) + list(range(ord("®"    )		,				ord("ÿ"    ) + 1    )    )
		)
		_snake_case      			=			bs[:]
		_snake_case      			=			0
		for b in range(2**8    ):
				if b not in bs:
						bs.append(lowercase_    )
						cs.append(2**8 + n    )
						n += 1
		_snake_case      			=			[chr(lowercase_    ) for n in cs]
		return dict(zip(lowercase_		,				lowercase_    )    )






def 						_lowercase						(      lowercase_    ):



		'''simple docstring'''







		_snake_case      			=			set()
		_snake_case      			=			word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)    )
				_snake_case      			=			char
		return pairs





class 	lowerCamelCase_				(			UpperCAmelCase_				):




					'''simple docstring'''






					lowercase__       =					VOCAB_FILES_NAMES
					lowercase__       =					PRETRAINED_VOCAB_FILES_MAP
					lowercase__       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowercase__       =					["""input_ids""", """attention_mask""", """token_type_ids"""]






					def __init__(       self,    __UpperCamelCase,    __UpperCamelCase,    __UpperCamelCase="replace",    __UpperCamelCase="[CLS]",    __UpperCamelCase="[SEP]",    __UpperCamelCase="[SEP]",    __UpperCamelCase="[CLS]",    __UpperCamelCase="[UNK]",    __UpperCamelCase="[PAD]",    __UpperCamelCase="[MASK]",    __UpperCamelCase=False,    __UpperCamelCase=False,    **__UpperCamelCase,    ):
							_snake_case      			=			AddedToken(__UpperCamelCase,    lstrip=__UpperCamelCase,    rstrip=__UpperCamelCase					) if isinstance(__UpperCamelCase,    __UpperCamelCase					) else bos_token
							_snake_case      			=			AddedToken(__UpperCamelCase,    lstrip=__UpperCamelCase,    rstrip=__UpperCamelCase					) if isinstance(__UpperCamelCase,    __UpperCamelCase					) else eos_token
							_snake_case      			=			AddedToken(__UpperCamelCase,    lstrip=__UpperCamelCase,    rstrip=__UpperCamelCase					) if isinstance(__UpperCamelCase,    __UpperCamelCase					) else sep_token
							_snake_case      			=			AddedToken(__UpperCamelCase,    lstrip=__UpperCamelCase,    rstrip=__UpperCamelCase					) if isinstance(__UpperCamelCase,    __UpperCamelCase					) else cls_token
							_snake_case      			=			AddedToken(__UpperCamelCase,    lstrip=__UpperCamelCase,    rstrip=__UpperCamelCase					) if isinstance(__UpperCamelCase,    __UpperCamelCase					) else unk_token
							_snake_case      			=			AddedToken(__UpperCamelCase,    lstrip=__UpperCamelCase,    rstrip=__UpperCamelCase					) if isinstance(__UpperCamelCase,    __UpperCamelCase					) else pad_token

							# Mask token behave like a normal word, i.e. include the space before it
							_snake_case      			=			AddedToken(__UpperCamelCase,    lstrip=__UpperCamelCase,    rstrip=__UpperCamelCase					) if isinstance(__UpperCamelCase,    __UpperCamelCase					) else mask_token

							super().__init__(
							    errors=__UpperCamelCase,    bos_token=__UpperCamelCase,    eos_token=__UpperCamelCase,    unk_token=__UpperCamelCase,    sep_token=__UpperCamelCase,    cls_token=__UpperCamelCase,    pad_token=__UpperCamelCase,    mask_token=__UpperCamelCase,    add_prefix_space=__UpperCamelCase,    add_bos_token=__UpperCamelCase,    **__UpperCamelCase,    )
							_snake_case      			=			add_bos_token

							with open(__UpperCamelCase,    encoding="utf-8"					) as vocab_handle:
									_snake_case      			=			json.load(__UpperCamelCase					)
							_snake_case      			=			{v: k for k, v in self.encoder.items()}
							_snake_case      			=			errors  # how to handle errors in decoding
							_snake_case      			=			bytes_to_unicode()
							_snake_case      			=			{v: k for k, v in self.byte_encoder.items()}
							with open(__UpperCamelCase,    encoding="utf-8"					) as merges_handle:
									_snake_case      			=			merges_handle.read().split("\n"					)[1:-1]
							_snake_case      			=			[tuple(merge.split()					) for merge in bpe_merges]
							_snake_case      			=			dict(zip(__UpperCamelCase,    range(len(__UpperCamelCase					)					)					)					)
							_snake_case      			=			{}
							_snake_case      			=			add_prefix_space

							# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
							_snake_case      			=			re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"					)






					@property
					# Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							return len(self.encoder					)






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							return dict(self.encoder,    **self.added_tokens_encoder					)






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase					):
							if token in self.cache:
									return self.cache[token]
							_snake_case      			=			tuple(__UpperCamelCase					)
							_snake_case      			=			get_pairs(__UpperCamelCase					)

							if not pairs:
									return token

							while True:
									_snake_case      			=			min(__UpperCamelCase,    key=lambda __UpperCamelCase					: self.bpe_ranks.get(__UpperCamelCase,    float("inf"					)					)					)
									if bigram not in self.bpe_ranks:
											break
									_snake_case			,       _snake_case      			=			bigram
									_snake_case      			=			[]
									_snake_case      			=			0
									while i < len(__UpperCamelCase					):
											try:
													_snake_case      			=			word.index(__UpperCamelCase,    __UpperCamelCase					)
											except ValueError:
													new_word.extend(word[i:]					)
													break
											else:
													new_word.extend(word[i:j]					)
													_snake_case      			=			j

											if word[i] == first and i < len(__UpperCamelCase					) - 1 and word[i + 1] == second:
													new_word.append(first + second					)
													i += 2
											else:
													new_word.append(word[i]					)
													i += 1
									_snake_case      			=			tuple(__UpperCamelCase					)
									_snake_case      			=			new_word
									if len(__UpperCamelCase					) == 1:
											break
									else:
											_snake_case      			=			get_pairs(__UpperCamelCase					)
							_snake_case      			=			" ".join(__UpperCamelCase					)
							_snake_case      			=			word
							return word






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase,    __UpperCamelCase = None					):
							if token_ids_a is None:
									return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							_snake_case      			=			[self.cls_token_id]
							_snake_case      			=			[self.sep_token_id]
							return cls + token_ids_a + sep + token_ids_a + sep






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase,    __UpperCamelCase = None,    __UpperCamelCase = False					):
							if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=__UpperCamelCase,    token_ids_a=__UpperCamelCase,    already_has_special_tokens=__UpperCamelCase					)

							if token_ids_a is None:
									return [1] + ([0] * len(__UpperCamelCase					)) + [1]
							return [1] + ([0] * len(__UpperCamelCase					)) + [1] + ([0] * len(__UpperCamelCase					)) + [1]






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase,    __UpperCamelCase = None					):
							_snake_case      			=			[self.sep_token_id]
							_snake_case      			=			[self.cls_token_id]

							if token_ids_a is None:
									return len(cls + token_ids_a + sep					) * [0]
							return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase					):
							_snake_case      			=			[]
							for token in re.findall(self.pat,    __UpperCamelCase					):
									_snake_case      			=			"".join(
									    self.byte_encoder[b] for b in token.encode("utf-8"					)					)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
									bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__UpperCamelCase					).split(" "					)					)
							return bpe_tokens






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase					):
							return self.encoder.get(__UpperCamelCase,    self.encoder.get(self.unk_token					)					)






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase					):
							return self.decoder.get(__UpperCamelCase					)






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase					):
							_snake_case      			=			"".join(__UpperCamelCase					)
							_snake_case      			=			bytearray([self.byte_decoder[c] for c in text]					).decode("utf-8",    errors=self.errors					)
							return text






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase,    __UpperCamelCase = None					):
							if not os.path.isdir(__UpperCamelCase					):
									logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''					)
									return
							_snake_case      			=			os.path.join(
							    __UpperCamelCase,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]					)
							_snake_case      			=			os.path.join(
							    __UpperCamelCase,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]					)

							with open(__UpperCamelCase,    "w",    encoding="utf-8"					) as f:
									f.write(json.dumps(self.encoder,    indent=2,    sort_keys=__UpperCamelCase,    ensure_ascii=__UpperCamelCase					) + "\n"					)

							_snake_case      			=			0
							with open(__UpperCamelCase,    "w",    encoding="utf-8"					) as writer:
									writer.write("#version: 0.2\n"					)
									for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),    key=lambda __UpperCamelCase					: kv[1]					):
											if index != token_index:
													logger.warning(
													    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
													    " Please check that the tokenizer is not corrupted!"					)
													_snake_case      			=			token_index
											writer.write(" ".join(__UpperCamelCase					) + "\n"					)
											index += 1

							return vocab_file, merge_file






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase,    __UpperCamelCase=False,    **__UpperCamelCase					):
							_snake_case      			=			kwargs.pop("add_prefix_space",    self.add_prefix_space					)
							if (is_split_into_words or add_prefix_space) and (len(__UpperCamelCase					) > 0 and not text[0].isspace()):
									_snake_case      			=			" " + text
							return (text, kwargs)






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase					):
							_snake_case      			=			[]
							for is_user, text in conversation.iter_texts():
									input_ids.extend(self.encode(__UpperCamelCase,    add_special_tokens=__UpperCamelCase					) + [self.eos_token_id]					)
							if len(__UpperCamelCase					) > self.model_max_length:
									_snake_case      			=			input_ids[-self.model_max_length :]
							return input_ids











'''simple docstring'''







from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_lowerCamelCase				:		List[str]				      =    logging.get_logger(__name__)

_lowerCamelCase				:		Optional[Any]				      =    {
    'MBZUAI/swiftformer-xs': 'https://huggingface.co/MBZUAI/swiftformer-xs/resolve/main/config.json',
}





class 	lowerCamelCase_				(			UpperCAmelCase_				):




					'''simple docstring'''






					lowercase__       =					"""swiftformer"""






					def __init__(       self,    __UpperCamelCase=3,    __UpperCamelCase=[3, 3, 6, 4],    __UpperCamelCase=[48, 56, 112, 220],    __UpperCamelCase=4,    __UpperCamelCase=[True, True, True, True],    __UpperCamelCase="gelu",    __UpperCamelCase=3,    __UpperCamelCase=2,    __UpperCamelCase=1,    __UpperCamelCase=0.0,    __UpperCamelCase=True,    __UpperCamelCase=1E-5,    __UpperCamelCase=1E-5,    **__UpperCamelCase,    ):
							super().__init__(**__UpperCamelCase					)
							_snake_case      			=			num_channels
							_snake_case      			=			depths
							_snake_case      			=			embed_dims
							_snake_case      			=			mlp_ratio
							_snake_case      			=			downsamples
							_snake_case      			=			hidden_act
							_snake_case      			=			down_patch_size
							_snake_case      			=			down_stride
							_snake_case      			=			down_pad
							_snake_case      			=			drop_path_rate
							_snake_case      			=			use_layer_scale
							_snake_case      			=			layer_scale_init_value
							_snake_case      			=			batch_norm_eps





class 	lowerCamelCase_				(			UpperCAmelCase_				):




					'''simple docstring'''






					lowercase__       =					version.parse("""1.11"""				)






					@property
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							return OrderedDict(
							    [
							        ("pixel_values", {0: "batch", 1: "num_channels", 2: "height", 3: "width"}),
							    ]					)






					@property
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							return 1E-4





