





"""simple docstring"""





import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
						from transformers.pipelines.conversational import Conversation

lowerCAmelCase__        =      logging.get_logger(__name__)

lowerCAmelCase__        =      {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

lowerCAmelCase__        =      {
    '''vocab_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/vocab.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/vocab.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/vocab.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/vocab.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/merges.txt''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/merges.txt''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/merges.txt''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/merges.txt''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/merges.txt''',
    },
}

lowerCAmelCase__        =      {
    '''gpt2''': 1_024,
    '''gpt2-medium''': 1_024,
    '''gpt2-large''': 1_024,
    '''gpt2-xl''': 1_024,
    '''distilgpt2''': 1_024,
}


@lru_cache()
def    a__       (       ):




		"""simple docstring"""


		UpperCamelCase =						(
		    list(range(ord("!"     )	,						ord("~"     ) + 1     )     ) + list(range(ord("¡"     )	,						ord("¬"     ) + 1     )     ) + list(range(ord("®"     )	,						ord("ÿ"     ) + 1     )     )
		)
		UpperCamelCase =						bs[:]
		UpperCamelCase =						0
		for b in range(2**8     ):
				if b not in bs:
						bs.append(_SCREAMING_SNAKE_CASE     )
						cs.append(2**8 + n     )
						n += 1
		UpperCamelCase =						[chr(_SCREAMING_SNAKE_CASE     ) for n in cs]
		return dict(zip(_SCREAMING_SNAKE_CASE	,						_SCREAMING_SNAKE_CASE     )     )


def    a__       (       _SCREAMING_SNAKE_CASE     ):




		"""simple docstring"""


		UpperCamelCase =						set()
		UpperCamelCase =						word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)     )
				UpperCamelCase =						char
		return pairs






class 							_lowerCamelCase   (							_lowercase			):
						UpperCAmelCase_			=      VOCAB_FILES_NAMES
						UpperCAmelCase_			=      PRETRAINED_VOCAB_FILES_MAP
						UpperCAmelCase_			=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCAmelCase_			=      ["input_ids", "attention_mask"]


						def __init__(self ,				__a ,				__a ,				__a="replace" ,				__a="<|endoftext|>" ,				__a="<|endoftext|>" ,				__a="<|endoftext|>" ,				__a=None ,				__a=False ,				__a=False ,				**__a ,				)			-> Optional[Any]:
								UpperCamelCase =						AddedToken(__a ,				lstrip=__a ,				rstrip=__a		) if isinstance(__a ,				__a		) else bos_token
								UpperCamelCase =						AddedToken(__a ,				lstrip=__a ,				rstrip=__a		) if isinstance(__a ,				__a		) else eos_token
								UpperCamelCase =						AddedToken(__a ,				lstrip=__a ,				rstrip=__a		) if isinstance(__a ,				__a		) else unk_token
								UpperCamelCase =						AddedToken(__a ,				lstrip=__a ,				rstrip=__a		) if isinstance(__a ,				__a		) else pad_token
								super().__init__(
								    errors=__a ,				unk_token=__a ,				bos_token=__a ,				eos_token=__a ,				pad_token=__a ,				add_prefix_space=__a ,				add_bos_token=__a ,				**__a ,				)
								UpperCamelCase =						add_bos_token

								with open(__a ,				encoding="utf-8"		) as vocab_handle:
										UpperCamelCase =						json.load(__a		)
								UpperCamelCase =						{v: k for k, v in self.encoder.items()}
								UpperCamelCase =						errors  # how to handle errors in decoding
								UpperCamelCase =						bytes_to_unicode()
								UpperCamelCase =						{v: k for k, v in self.byte_encoder.items()}
								with open(__a ,				encoding="utf-8"		) as merges_handle:
										UpperCamelCase =						merges_handle.read().split("\n"		)[1:-1]
								UpperCamelCase =						[tuple(merge.split()		) for merge in bpe_merges]
								UpperCamelCase =						dict(zip(__a ,				range(len(__a		)		)		)		)
								UpperCamelCase =						{}
								UpperCamelCase =						add_prefix_space

								# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
								UpperCamelCase =						re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"		)


						@property
						def      snake_case_ (self		)			-> Optional[Any]:
								return len(self.encoder		)


						def      snake_case_ (self		)			-> str:
								return dict(self.encoder ,				**self.added_tokens_encoder		)


						def      snake_case_ (self ,				__a		)			-> int:
								if token in self.cache:
										return self.cache[token]
								UpperCamelCase =						tuple(__a		)
								UpperCamelCase =						get_pairs(__a		)

								if not pairs:
										return token

								while True:
										UpperCamelCase =						min(__a ,				key=lambda __a		: self.bpe_ranks.get(__a ,				float("inf"		)		)		)
										if bigram not in self.bpe_ranks:
												break
										UpperCamelCase ,					UpperCamelCase =						bigram
										UpperCamelCase =						[]
										UpperCamelCase =						0
										while i < len(__a		):
												try:
														UpperCamelCase =						word.index(__a ,				__a		)
												except ValueError:
														new_word.extend(word[i:]		)
														break
												else:
														new_word.extend(word[i:j]		)
														UpperCamelCase =						j

												if word[i] == first and i < len(__a		) - 1 and word[i + 1] == second:
														new_word.append(first + second		)
														i += 2
												else:
														new_word.append(word[i]		)
														i += 1
										UpperCamelCase =						tuple(__a		)
										UpperCamelCase =						new_word
										if len(__a		) == 1:
												break
										else:
												UpperCamelCase =						get_pairs(__a		)
								UpperCamelCase =						" ".join(__a		)
								UpperCamelCase =						word
								return word


						def      snake_case_ (self ,				__a ,				__a=None		)			-> Tuple:
								if self.add_bos_token:
										UpperCamelCase =						[self.bos_token_id]
								else:
										UpperCamelCase =						[]

								UpperCamelCase =						bos_token_ids + token_ids_a

								if token_ids_a is None:
										return output

								return output + bos_token_ids + token_ids_a


						def      snake_case_ (self ,				__a ,				__a = None ,				__a = False		)			-> List[int]:
								if already_has_special_tokens:
										return super().get_special_tokens_mask(
										    token_ids_a=__a ,				token_ids_a=__a ,				already_has_special_tokens=__a		)

								if not self.add_bos_token:
										return super().get_special_tokens_mask(
										    token_ids_a=__a ,				token_ids_a=__a ,				already_has_special_tokens=__a		)

								if token_ids_a is None:
										return [1] + ([0] * len(__a		))
								return [1] + ([0] * len(__a		)) + [1] + ([0] * len(__a		))


						def      snake_case_ (self ,				__a		)			-> str:
								UpperCamelCase =						[]
								for token in re.findall(self.pat ,				__a		):
										UpperCamelCase =						"".join(
										    self.byte_encoder[b] for b in token.encode("utf-8"		)		)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
										bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__a		).split(" "		)		)
								return bpe_tokens


						def      snake_case_ (self ,				__a		)			-> List[str]:
								return self.encoder.get(__a ,				self.encoder.get(self.unk_token		)		)


						def      snake_case_ (self ,				__a		)			-> Dict:
								return self.decoder.get(__a		)


						def      snake_case_ (self ,				__a		)			-> List[Any]:
								UpperCamelCase =						"".join(__a		)
								UpperCamelCase =						bytearray([self.byte_decoder[c] for c in text]		).decode("utf-8" ,				errors=self.errors		)
								return text


						def      snake_case_ (self ,				__a ,				__a = None		)			-> Tuple[str]:
								if not os.path.isdir(__a		):
										logger.error(F"Vocabulary path ({save_directory}) should be a directory"		)
										return
								UpperCamelCase =						os.path.join(
								    __a ,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
								UpperCamelCase =						os.path.join(
								    __a ,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]		)

								with open(__a ,				"w" ,				encoding="utf-8"		) as f:
										f.write(json.dumps(self.encoder ,				indent=2 ,				sort_keys=__a ,				ensure_ascii=__a		) + "\n"		)

								UpperCamelCase =						0
								with open(__a ,				"w" ,				encoding="utf-8"		) as writer:
										writer.write("#version: 0.2\n"		)
										for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,				key=lambda __a		: kv[1]		):
												if index != token_index:
														logger.warning(
														    F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
														    " Please check that the tokenizer is not corrupted!"		)
														UpperCamelCase =						token_index
												writer.write(" ".join(__a		) + "\n"		)
												index += 1

								return vocab_file, merge_file


						def      snake_case_ (self ,				__a ,				__a=False ,				**__a		)			-> str:
								UpperCamelCase =						kwargs.pop("add_prefix_space" ,				self.add_prefix_space		)
								if is_split_into_words or add_prefix_space:
										UpperCamelCase =						" " + text
								return (text, kwargs)




						def      snake_case_ (self ,				__a		)			-> List[int]:
								UpperCamelCase =						[]
								for is_user, text in conversation.iter_texts():
										input_ids.extend(self.encode(__a ,				add_special_tokens=__a		) + [self.eos_token_id]		)
								if len(__a		) > self.model_max_length:
										UpperCamelCase =						input_ids[-self.model_max_length :]
								return input_ids













"""simple docstring"""





from ..utils import DummyObject, requires_backends





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Union[str, Any]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Any:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Dict:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Optional[Any]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Tuple:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Union[str, Any]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Optional[int]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Optional[Any]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Union[str, Any]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> int:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> List[Any]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Tuple:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Dict:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Any:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Optional[Any]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Dict:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Any:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Dict:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> List[str]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Tuple:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Tuple:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> int:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Optional[Any]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> int:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Tuple:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Tuple:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> str:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Optional[int]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Optional[Any]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Union[str, Any]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> int:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Any:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Dict:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Optional[int]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Optional[int]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Union[str, Any]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> int:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> List[Any]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Dict:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Dict:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> List[Any]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Optional[int]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Tuple:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> List[Any]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> List[str]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> str:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> int:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Dict:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> List[Any]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> List[str]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> int:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Optional[int]:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> int:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> str:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> str:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> int:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Dict:
								requires_backends(self ,				["tokenizers"]		)





class 							_lowerCamelCase   (							metaclass=_lowercase			):
						UpperCAmelCase_			=      ["tokenizers"]


						def __init__(self ,				*__a ,				**__a		)			-> Dict:
								requires_backends(self ,				["tokenizers"]		)






