


'''simple docstring'''




import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
				from transformers.pipelines.conversational import Conversation

__lowercase	:   str   		=					logging.get_logger(__name__)

__lowercase	:   Tuple   		=					{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

__lowercase	:   Dict   		=					{
    """vocab_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"""
        ),
    },
}

__lowercase	:   List[str]   		=					{
    """microsoft/deberta-base""": 5_12,
    """microsoft/deberta-large""": 5_12,
    """microsoft/deberta-xlarge""": 5_12,
    """microsoft/deberta-base-mnli""": 5_12,
    """microsoft/deberta-large-mnli""": 5_12,
    """microsoft/deberta-xlarge-mnli""": 5_12,
}

__lowercase	:   List[str]   		=					{
    """microsoft/deberta-base""": {"""do_lower_case""": False},
    """microsoft/deberta-large""": {"""do_lower_case""": False},
}





def 				a_							(    )      ->Union[str, Any]:






				'''simple docstring'''

				__lowerCamelCase		:   List[Any]           =       (
				    list(range(ord("!"     )				,    ord("~"     ) + 1     )     ) + list(range(ord("¡"     )				,    ord("¬"     ) + 1     )     ) + list(range(ord("®"     )				,    ord("ÿ"     ) + 1     )     )
				)
				__lowerCamelCase		:   int           =       bs[:]
				__lowerCamelCase		:   List[str]           =       0
				for b in range(2**8     ):
								if b not in bs:
												bs.append(lowercase     )
												cs.append(2**8 + n     )
												n += 1
				__lowerCamelCase		:   Union[str, Any]           =       [chr(lowercase     ) for n in cs]
				return dict(zip(lowercase				,    lowercase     )     )





def 				a_							(    lowercase     )      ->Tuple:






				'''simple docstring'''

				__lowerCamelCase		:   int           =       set()
				__lowerCamelCase		:   Optional[Any]           =       word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)     )
								__lowerCamelCase		:   int           =       char
				return pairs





class _snake_case (			a__ ):
							"""simple docstring"""





							__a				       =  VOCAB_FILES_NAMES
							__a				       =  PRETRAINED_VOCAB_FILES_MAP
							__a				       =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__a				       =  ['''input_ids''', '''attention_mask''', '''token_type_ids''']







							def __init__( self      ,					__A      ,					__A      ,					__A="replace"      ,					__A="[CLS]"      ,					__A="[SEP]"      ,					__A="[SEP]"      ,					__A="[CLS]"      ,					__A="[UNK]"      ,					__A="[PAD]"      ,					__A="[MASK]"      ,					__A=False      ,					__A=False      ,					**__A      ,					)  ->int:



											"""simple docstring"""







											__lowerCamelCase		:   List[str]           =       AddedToken(__A      ,					lstrip=__A      ,					rstrip=__A						) if isinstance(__A      ,					__A						) else bos_token
											__lowerCamelCase		:   Optional[Any]           =       AddedToken(__A      ,					lstrip=__A      ,					rstrip=__A						) if isinstance(__A      ,					__A						) else eos_token
											__lowerCamelCase		:   int           =       AddedToken(__A      ,					lstrip=__A      ,					rstrip=__A						) if isinstance(__A      ,					__A						) else sep_token
											__lowerCamelCase		:   Any           =       AddedToken(__A      ,					lstrip=__A      ,					rstrip=__A						) if isinstance(__A      ,					__A						) else cls_token
											__lowerCamelCase		:   Any           =       AddedToken(__A      ,					lstrip=__A      ,					rstrip=__A						) if isinstance(__A      ,					__A						) else unk_token
											__lowerCamelCase		:   Optional[int]           =       AddedToken(__A      ,					lstrip=__A      ,					rstrip=__A						) if isinstance(__A      ,					__A						) else pad_token

											# Mask token behave like a normal word, i.e. include the space before it
											__lowerCamelCase		:   Any           =       AddedToken(__A      ,					lstrip=__A      ,					rstrip=__A						) if isinstance(__A      ,					__A						) else mask_token

											super().__init__(
											    errors=__A      ,					bos_token=__A      ,					eos_token=__A      ,					unk_token=__A      ,					sep_token=__A      ,					cls_token=__A      ,					pad_token=__A      ,					mask_token=__A      ,					add_prefix_space=__A      ,					add_bos_token=__A      ,					**__A      ,					)
											__lowerCamelCase		:   List[Any]           =       add_bos_token

											with open(__A      ,					encoding="utf-8"						) as vocab_handle:
															__lowerCamelCase		:   List[Any]           =       json.load(__A						)
											__lowerCamelCase		:   int           =       {v: k for k, v in self.encoder.items()}
											__lowerCamelCase		:   Dict           =       errors  # how to handle errors in decoding
											__lowerCamelCase		:   Union[str, Any]           =       bytes_to_unicode()
											__lowerCamelCase		:   Optional[Any]           =       {v: k for k, v in self.byte_encoder.items()}
											with open(__A      ,					encoding="utf-8"						) as merges_handle:
															__lowerCamelCase		:   Dict           =       merges_handle.read().split("\n"						)[1:-1]
											__lowerCamelCase		:   List[str]           =       [tuple(merge.split()						) for merge in bpe_merges]
											__lowerCamelCase		:   List[Any]           =       dict(zip(__A      ,					range(len(__A						)						)						)						)
											__lowerCamelCase		:   List[Any]           =       {}
											__lowerCamelCase		:   Tuple           =       add_prefix_space

											# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
											__lowerCamelCase		:   Optional[int]           =       re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"						)







							@property
							# Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
							def 		__SCREAMING_SNAKE_CASE( self						)  ->Union[str, Any]:



											"""simple docstring"""







											return len(self.encoder						)







							def 		__SCREAMING_SNAKE_CASE( self						)  ->Optional[int]:



											"""simple docstring"""







											return dict(self.encoder      ,					**self.added_tokens_encoder						)







							def 		__SCREAMING_SNAKE_CASE( self      ,					__A						)  ->Union[str, Any]:



											"""simple docstring"""







											if token in self.cache:
															return self.cache[token]
											__lowerCamelCase		:   Tuple           =       tuple(__A						)
											__lowerCamelCase		:   Dict           =       get_pairs(__A						)

											if not pairs:
															return token

											while True:
															__lowerCamelCase		:   List[Any]           =       min(__A      ,					key=lambda __A						: self.bpe_ranks.get(__A      ,					float("inf"						)						)						)
															if bigram not in self.bpe_ranks:
																			break
															__lowerCamelCase    ,       __lowerCamelCase		:   List[Any]           =       bigram
															__lowerCamelCase		:   Union[str, Any]           =       []
															__lowerCamelCase		:   str           =       0
															while i < len(__A						):
																			try:
																							__lowerCamelCase		:   Optional[int]           =       word.index(__A      ,					__A						)
																			except ValueError:
																							new_word.extend(word[i:]						)
																							break
																			else:
																							new_word.extend(word[i:j]						)
																							__lowerCamelCase		:   List[str]           =       j

																			if word[i] == first and i < len(__A						) - 1 and word[i + 1] == second:
																							new_word.append(first + second						)
																							i += 2
																			else:
																							new_word.append(word[i]						)
																							i += 1
															__lowerCamelCase		:   Optional[int]           =       tuple(__A						)
															__lowerCamelCase		:   Optional[Any]           =       new_word
															if len(__A						) == 1:
																			break
															else:
																			__lowerCamelCase		:   Dict           =       get_pairs(__A						)
											__lowerCamelCase		:   Tuple           =       " ".join(__A						)
											__lowerCamelCase		:   Union[str, Any]           =       word
											return word







							def 		__SCREAMING_SNAKE_CASE( self      ,					__A      ,					__A = None						)  ->List[int]:



											"""simple docstring"""







											if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
											__lowerCamelCase		:   List[Any]           =       [self.cls_token_id]
											__lowerCamelCase		:   str           =       [self.sep_token_id]
											return cls + token_ids_a + sep + token_ids_a + sep







							def 		__SCREAMING_SNAKE_CASE( self      ,					__A      ,					__A = None      ,					__A = False						)  ->List[int]:



											"""simple docstring"""







											if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=__A      ,					token_ids_a=__A      ,					already_has_special_tokens=__A						)

											if token_ids_a is None:
															return [1] + ([0] * len(__A						)) + [1]
											return [1] + ([0] * len(__A						)) + [1] + ([0] * len(__A						)) + [1]







							def 		__SCREAMING_SNAKE_CASE( self      ,					__A      ,					__A = None						)  ->List[int]:



											"""simple docstring"""







											__lowerCamelCase		:   str           =       [self.sep_token_id]
											__lowerCamelCase		:   Any           =       [self.cls_token_id]

											if token_ids_a is None:
															return len(cls + token_ids_a + sep						) * [0]
											return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]







							def 		__SCREAMING_SNAKE_CASE( self      ,					__A						)  ->Tuple:



											"""simple docstring"""







											__lowerCamelCase		:   Dict           =       []
											for token in re.findall(self.pat      ,					__A						):
															__lowerCamelCase		:   str           =       "".join(
															    self.byte_encoder[b] for b in token.encode("utf-8"						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
															bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__A						).split(" "						)						)
											return bpe_tokens







							def 		__SCREAMING_SNAKE_CASE( self      ,					__A						)  ->str:



											"""simple docstring"""







											return self.encoder.get(__A      ,					self.encoder.get(self.unk_token						)						)







							def 		__SCREAMING_SNAKE_CASE( self      ,					__A						)  ->Dict:



											"""simple docstring"""







											return self.decoder.get(__A						)







							def 		__SCREAMING_SNAKE_CASE( self      ,					__A						)  ->List[str]:



											"""simple docstring"""







											__lowerCamelCase		:   Any           =       "".join(__A						)
											__lowerCamelCase		:   Dict           =       bytearray([self.byte_decoder[c] for c in text]						).decode("utf-8"      ,					errors=self.errors						)
											return text







							def 		__SCREAMING_SNAKE_CASE( self      ,					__A      ,					__A = None						)  ->Tuple[str]:



											"""simple docstring"""







											if not os.path.isdir(__A						):
															logger.error(f'Vocabulary path ({save_directory}) should be a directory'						)
															return
											__lowerCamelCase		:   str           =       os.path.join(
											    __A      ,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)
											__lowerCamelCase		:   Any           =       os.path.join(
											    __A      ,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]						)

											with open(__A      ,					"w"      ,					encoding="utf-8"						) as f:
															f.write(json.dumps(self.encoder      ,					indent=2      ,					sort_keys=__A      ,					ensure_ascii=__A						) + "\n"						)

											__lowerCamelCase		:   List[Any]           =       0
											with open(__A      ,					"w"      ,					encoding="utf-8"						) as writer:
															writer.write("#version: 0.2\n"						)
															for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,					key=lambda __A						: kv[1]						):
																			if index != token_index:
																							logger.warning(
																							    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																							    " Please check that the tokenizer is not corrupted!"						)
																							__lowerCamelCase		:   Optional[int]           =       token_index
																			writer.write(" ".join(__A						) + "\n"						)
																			index += 1

											return vocab_file, merge_file







							def 		__SCREAMING_SNAKE_CASE( self      ,					__A      ,					__A=False      ,					**__A						)  ->int:



											"""simple docstring"""







											__lowerCamelCase		:   Any           =       kwargs.pop("add_prefix_space"      ,					self.add_prefix_space						)
											if (is_split_into_words or add_prefix_space) and (len(__A						) > 0 and not text[0].isspace()):
															__lowerCamelCase		:   Union[str, Any]           =       " " + text
											return (text, kwargs)







							def 		__SCREAMING_SNAKE_CASE( self      ,					__A						)  ->List[int]:



											"""simple docstring"""







											__lowerCamelCase		:   Optional[int]           =       []
											for is_user, text in conversation.iter_texts():
															input_ids.extend(self.encode(__A      ,					add_special_tokens=__A						) + [self.eos_token_id]						)
											if len(__A						) > self.model_max_length:
															__lowerCamelCase		:   Any           =       input_ids[-self.model_max_length :]
											return input_ids









'''simple docstring'''




import argparse
import copy
import json
import multiprocessing
import os
import time

from create_dummy_models import COMPOSITE_MODELS, create_tiny_models
from huggingface_hub import ModelFilter, hf_api

import transformers
from transformers import AutoFeatureExtractor, AutoImageProcessor, AutoTokenizer
from transformers.image_processing_utils import BaseImageProcessor





def 				a_							(    )      ->List[Any]:






				'''simple docstring'''

				__lowerCamelCase		:   Optional[Any]           =       set()
				# Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
				for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
								__lowerCamelCase		:   Dict           =       getattr(transformers.models.auto				,    lowercase				,    lowercase     )
								if module is None:
												continue
								# all mappings in a single auto modeling file
								__lowerCamelCase		:   Any           =       [
								    x
								    for x in dir(lowercase     )
								    if x.endswith("_MAPPING_NAMES"     )
								    and (x.startswith("MODEL_"     ) or x.startswith("TF_MODEL_"     ) or x.startswith("FLAX_MODEL_"     ))
								]
								for name in mapping_names:
												__lowerCamelCase		:   Union[str, Any]           =       getattr(lowercase				,    lowercase     )
												if mapping is not None:
																for v in mapping.values():
																				if isinstance(lowercase				,    (list, tuple)     ):
																								model_names.update(lowercase     )
																				elif isinstance(lowercase				,    lowercase     ):
																								model_names.add(lowercase     )

				return sorted(lowercase     )





def 				a_							(    )      ->int:






				'''simple docstring'''

				__lowerCamelCase		:   Tuple           =       set(get_all_model_names()     )

				with open("tests/utils/tiny_model_summary.json"     ) as fp:
								__lowerCamelCase		:   Optional[int]           =       json.load(lowercase     )
				__lowerCamelCase		:   Union[str, Any]           =       set()
				for model_base_name in tiny_model_info:
								tiny_models_names.update(tiny_model_info[model_base_name]["model_classes"]     )

				# Remove a tiny model name if one of its framework implementation hasn't yet a tiny version on the Hub.
				__lowerCamelCase		:   List[str]           =       model_names.difference(lowercase     )
				for model_name in copy.copy(lowercase     ):
								if not model_name.startswith("TF"     ) and F'TF{model_name}' in not_on_hub:
												tiny_models_names.remove(lowercase     )
								elif model_name.startswith("TF"     ) and model_name[2:] in not_on_hub:
												tiny_models_names.remove(lowercase     )

				return sorted(lowercase     )





def 				a_							(    lowercase     )      ->str:






				'''simple docstring'''

				__lowerCamelCase		:   Optional[int]           =       COMPOSITE_MODELS.values()

				# All tiny model base names on Hub
				__lowerCamelCase		:   List[Any]           =       get_all_model_names()
				__lowerCamelCase		:   Any           =       hf_api.list_models(
				    filter=ModelFilter(
				        author="hf-internal-testing"				,    )     )
				__lowerCamelCase		:   str           =       set()
				for x in models:
								__lowerCamelCase		:   Any           =       x.modelId
								__lowerCamelCase    ,       __lowerCamelCase		:   List[Any]           =       model.split("/"     )
								if not model.startswith("tiny-random-"     ):
												continue
								__lowerCamelCase		:   int           =       model.replace("tiny-random-"				,    ""     )
								if not model[0].isupper():
												continue
								if model not in model_names and model not in special_models:
												continue
								_models.add(lowercase     )

				__lowerCamelCase		:   Tuple           =       sorted(_models     )
				# All tiny model names on Hub
				__lowerCamelCase		:   Union[str, Any]           =       {}
				for model in models:
								__lowerCamelCase		:   List[str]           =       F'hf-internal-testing/tiny-random-{model}'
								__lowerCamelCase		:   Any           =       model.split("-"     )[0]
								try:
												__lowerCamelCase		:   str           =       hf_api.repo_info(lowercase     )
												__lowerCamelCase		:   str           =       {
												    "tokenizer_classes": set(),
												    "processor_classes": set(),
												    "model_classes": set(),
												    "sha": repo_info.sha,
												}
								except Exception:
												continue
								try:
												time.sleep(1     )
												__lowerCamelCase		:   Optional[int]           =       AutoTokenizer.from_pretrained(lowercase     )
												content["tokenizer_classes"].add(tokenizer_fast.__class__.__name__     )
								except Exception:
												pass
								try:
												time.sleep(1     )
												__lowerCamelCase		:   List[Any]           =       AutoTokenizer.from_pretrained(lowercase				,    use_fast=lowercase     )
												content["tokenizer_classes"].add(tokenizer_slow.__class__.__name__     )
								except Exception:
												pass
								try:
												time.sleep(1     )
												__lowerCamelCase		:   List[Any]           =       AutoImageProcessor.from_pretrained(lowercase     )
												content["processor_classes"].add(img_p.__class__.__name__     )
								except Exception:
												pass
								try:
												time.sleep(1     )
												__lowerCamelCase		:   Tuple           =       AutoFeatureExtractor.from_pretrained(lowercase     )
												if not isinstance(lowercase				,    lowercase     ):
																content["processor_classes"].add(feat_p.__class__.__name__     )
								except Exception:
												pass
								try:
												time.sleep(1     )
												__lowerCamelCase		:   Optional[Any]           =       getattr(lowercase				,    lowercase     )
												__lowerCamelCase		:   Optional[Any]           =       model_class.from_pretrained(lowercase     )
												content["model_classes"].add(m.__class__.__name__     )
								except Exception:
												pass
								try:
												time.sleep(1     )
												__lowerCamelCase		:   List[str]           =       getattr(lowercase				,    F'TF{model}'     )
												__lowerCamelCase		:   Dict           =       model_class.from_pretrained(lowercase     )
												content["model_classes"].add(m.__class__.__name__     )
								except Exception:
												pass

								__lowerCamelCase		:   List[Any]           =       sorted(content["tokenizer_classes"]     )
								__lowerCamelCase		:   str           =       sorted(content["processor_classes"]     )
								__lowerCamelCase		:   Union[str, Any]           =       sorted(content["model_classes"]     )

								__lowerCamelCase		:   Tuple           =       content
								with open(os.path.join(lowercase				,    "hub_tiny_model_summary.json"     )				,    "w"     ) as fp:
												json.dump(lowercase				,    lowercase				,    ensure_ascii=lowercase				,    indent=4     )


if __name__ == "__main__":
				__lowercase	:   Any   		=					argparse.ArgumentParser()
				parser.add_argument("""--num_workers""", default=1, type=int, help="""The number of workers to run.""")
				__lowercase	:   Optional[int]   		=					parser.parse_args()

				# This has to be `spawn` to avoid hanging forever!
				multiprocessing.set_start_method("""spawn""")

				__lowercase	:   str   		=					"""tiny_models"""
				__lowercase	:   List[Any]   		=					True
				__lowercase	:   Optional[int]   		=					None
				__lowercase	:   List[Any]   		=					get_tiny_model_names_from_repo()
				__lowercase	:   List[Any]   		=					True
				__lowercase	:   Dict   		=					True
				__lowercase	:   Tuple   		=					"""hf-internal-testing"""

				create_tiny_models(
				    output_path,
				    all,
				    model_types,
				    models_to_skip,
				    no_check,
				    upload,
				    organization,
				    token=os.environ.get("""TOKEN""", None),
				    num_workers=args.num_workers,
				)





