



import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


__SCREAMING_SNAKE_CASE     =	logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE     =	{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

__SCREAMING_SNAKE_CASE     =	{
    """vocab_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/vocab.json"""},
    """merges_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/merges.txt"""},
}

__SCREAMING_SNAKE_CASE     =	{
    """openai-gpt""": 512,
}




def 						UpperCAmelCase (    _lowerCamelCase     ):
			A				:       Any														=    text.strip()
			if not text:
						return []
			A				:       Any														=    text.split()
			return tokens

class 						lowerCamelCase_     (			_A    ):






						'''simple docstring'''







						def __init__(						self   :      Optional[int]						,						__lowerCamelCase   :      Tuple=True						,						__lowerCamelCase   :      int=None						,						__lowerCamelCase   :      Tuple=True						,						__lowerCamelCase   :      Optional[Any]=None						,						__lowerCamelCase   :      Optional[int]=True						,						)   ->					Optional[Any]:
									if never_split is None:
												A				:       Any														=    []
									A				:       Optional[int]														=    do_lower_case
									A				:       Any														=    set(__lowerCamelCase			)
									A				:       List[Any]														=    tokenize_chinese_chars
									A				:       Tuple														=    strip_accents
									A				:       List[Any]														=    do_split_on_punc







						def       SCREAMING_SNAKE_CASE__		(						self   :      str						,						__lowerCamelCase   :      List[Any]						,						__lowerCamelCase   :      Dict=None			)   ->					Optional[int]:
									A				:       Any														=    self.never_split.union(set(__lowerCamelCase			)			) if never_split else self.never_split
									A				:       List[str]														=    self._clean_text(__lowerCamelCase			)

									# This was added on November 1st, 2018 for the multilingual and Chinese
									# models. This is also applied to the English models now, but it doesn't
									# matter since the English models were not trained on any Chinese data
									# and generally don't have any Chinese data in them (there are Chinese
									# characters in the vocabulary because Wikipedia does have some Chinese
									# words in the English Wikipedia.).
									if self.tokenize_chinese_chars:
												A				:       int														=    self._tokenize_chinese_chars(__lowerCamelCase			)
									# prevents treating the same character with different unicode codepoints as different characters
									A				:       Optional[int]														=    unicodedata.normalize("NFC"						,						__lowerCamelCase			)
									A				:       Dict														=    whitespace_tokenize(__lowerCamelCase			)
									A				:       List[Any]														=    []
									for token in orig_tokens:
												if token not in never_split:
															if self.do_lower_case:
																		A				:       List[str]														=    token.lower()
																		if self.strip_accents is not False:
																					A				:       int														=    self._run_strip_accents(__lowerCamelCase			)
															elif self.strip_accents:
																		A				:       Dict														=    self._run_strip_accents(__lowerCamelCase			)
												split_tokens.extend(self._run_split_on_punc(__lowerCamelCase						,						__lowerCamelCase			)			)

									A				:       Optional[int]														=    whitespace_tokenize(" ".join(__lowerCamelCase			)			)
									return output_tokens







						def       SCREAMING_SNAKE_CASE__		(						self   :      List[Any]						,						__lowerCamelCase   :      List[Any]			)   ->					Union[str, Any]:
									A				:       Any														=    unicodedata.normalize("NFD"						,						__lowerCamelCase			)
									A				:       Union[str, Any]														=    []
									for char in text:
												A				:       List[str]														=    unicodedata.category(__lowerCamelCase			)
												if cat == "Mn":
															continue
												output.append(__lowerCamelCase			)
									return "".join(__lowerCamelCase			)







						def       SCREAMING_SNAKE_CASE__		(						self   :      Tuple						,						__lowerCamelCase   :      Any						,						__lowerCamelCase   :      List[Any]=None			)   ->					Tuple:
									if not self.do_split_on_punc or (never_split is not None and text in never_split):
												return [text]
									A				:       Optional[Any]														=    list(__lowerCamelCase			)
									A				:       List[Any]														=    0
									A				:       List[str]														=    True
									A				:       int														=    []
									while i < len(__lowerCamelCase			):
												A				:       Dict														=    chars[i]
												if _is_punctuation(__lowerCamelCase			):
															output.append([char]			)
															A				:       List[Any]														=    True
												else:
															if start_new_word:
																		output.append([]			)
															A				:       Any														=    False
															output[-1].append(__lowerCamelCase			)
												i += 1

									return ["".join(__lowerCamelCase			) for x in output]







						def       SCREAMING_SNAKE_CASE__		(						self   :      Optional[Any]						,						__lowerCamelCase   :      Any			)   ->					int:
									A				:       Tuple														=    []
									for char in text:
												A				:       Optional[Any]														=    ord(__lowerCamelCase			)
												if self._is_chinese_char(__lowerCamelCase			):
															output.append(" "			)
															output.append(__lowerCamelCase			)
															output.append(" "			)
												else:
															output.append(__lowerCamelCase			)
									return "".join(__lowerCamelCase			)







						def       SCREAMING_SNAKE_CASE__		(						self   :      Dict						,						__lowerCamelCase   :      int			)   ->					Any:
									# This defines a "chinese character" as anything in the CJK Unicode block:
									#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
									#
									# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
									# despite its name. The modern Korean Hangul alphabet is a different block,
									# as is Japanese Hiragana and Katakana. Those alphabets are used to write
									# space-separated words, so they are not treated specially and handled
									# like the all of the other languages.
									if (
									    (cp >= 0X4_e_0_0 and cp <= 0X9_f_f_f)
									    or (cp >= 0X3_4_0_0 and cp <= 0X4_d_b_f)  #
									    or (cp >= 0X2_0_0_0_0 and cp <= 0X2_a_6_d_f)  #
									    or (cp >= 0X2_a_7_0_0 and cp <= 0X2_b_7_3_f)  #
									    or (cp >= 0X2_b_7_4_0 and cp <= 0X2_b_8_1_f)  #
									    or (cp >= 0X2_b_8_2_0 and cp <= 0X2_c_e_a_f)  #
									    or (cp >= 0Xf_9_0_0 and cp <= 0Xf_a_f_f)
									    or (cp >= 0X2_f_8_0_0 and cp <= 0X2_f_a_1_f)  #
									):  #
												return True

									return False







						def       SCREAMING_SNAKE_CASE__		(						self   :      Dict						,						__lowerCamelCase   :      Optional[Any]			)   ->					List[Any]:
									A				:       Any														=    []
									for char in text:
												A				:       Dict														=    ord(__lowerCamelCase			)
												if cp == 0 or cp == 0Xf_f_f_d or _is_control(__lowerCamelCase			):
															continue
												if _is_whitespace(__lowerCamelCase			):
															output.append(" "			)
												else:
															output.append(__lowerCamelCase			)
									return "".join(__lowerCamelCase			)







def 						UpperCAmelCase (    _lowerCamelCase     ):
			A				:       Tuple														=    set()
			A				:       List[str]														=    word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)     )
						A				:       Tuple														=    char
			return pairs




def 						UpperCAmelCase (    _lowerCamelCase     ):
			A				:       Dict														=    text.replace("—"			,     "-"     )
			A				:       Tuple														=    text.replace("–"			,     "-"     )
			A				:       Tuple														=    text.replace("―"			,     "-"     )
			A				:       Optional[Any]														=    text.replace("…"			,     "..."     )
			A				:       Union[str, Any]														=    text.replace("´"			,     "'"     )
			A				:       Optional[Any]														=    re.sub(R"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"			,     R" \1 "			,     _lowerCamelCase     )
			A				:       Dict														=    re.sub(R"\s*\n\s*"			,     " \n "			,     _lowerCamelCase     )
			A				:       str														=    re.sub(R"[^\S\n]+"			,     " "			,     _lowerCamelCase     )
			return text.strip()






class 						lowerCamelCase_     (			_A    ):






						'''simple docstring'''
						a__ =	VOCAB_FILES_NAMES
						a__ =	PRETRAINED_VOCAB_FILES_MAP
						a__ =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						a__ =	["input_ids", "attention_mask"]







						def __init__(						self   :      int						,						__lowerCamelCase   :      Tuple						,						__lowerCamelCase   :      str						,						__lowerCamelCase   :      Optional[int]="<unk>"						,						**__lowerCamelCase   :      int			)   ->					Dict:
									super().__init__(unk_token=__lowerCamelCase						,						**__lowerCamelCase			)

									try:
												import ftfy
												from spacy.lang.en import English

												A				:       Optional[Any]														=    English()
												A				:       int														=    _nlp.tokenizer
												A				:       Union[str, Any]														=    ftfy.fix_text
									except ImportError:
												logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."			)
												A				:       Dict														=    BasicTokenizer(do_lower_case=__lowerCamelCase			)
												A				:       List[Any]														=    None

									with open(__lowerCamelCase						,						encoding="utf-8"			) as vocab_handle:
												A				:       int														=    json.load(__lowerCamelCase			)
									A				:       Optional[int]														=    {v: k for k, v in self.encoder.items()}
									with open(__lowerCamelCase						,						encoding="utf-8"			) as merges_handle:
												A				:       int														=    merges_handle.read().split("\n"			)[1:-1]
									A				:       Union[str, Any]														=    [tuple(merge.split()			) for merge in merges]
									A				:       List[str]														=    dict(zip(__lowerCamelCase						,						range(len(__lowerCamelCase			)			)			)			)
									A				:       List[Any]														=    {}







						@property
						def       SCREAMING_SNAKE_CASE__		(						self   :      Dict			)   ->					Optional[Any]:
									return True







						@property
						def       SCREAMING_SNAKE_CASE__		(						self   :      Optional[int]			)   ->					Any:
									return len(self.encoder			)







						def       SCREAMING_SNAKE_CASE__		(						self   :      List[str]			)   ->					List[Any]:
									return dict(self.encoder						,						**self.added_tokens_encoder			)







						def       SCREAMING_SNAKE_CASE__		(						self   :      Union[str, Any]						,						__lowerCamelCase   :      Optional[Any]			)   ->					List[Any]:
									A				:       Optional[Any]														=    tuple(token[:-1]			) + (token[-1] + "</w>",)
									if token in self.cache:
												return self.cache[token]
									A				:       List[str]														=    get_pairs(__lowerCamelCase			)

									if not pairs:
												return token + "</w>"

									while True:
												A				:       Dict														=    min(__lowerCamelCase						,						key=lambda __lowerCamelCase			: self.bpe_ranks.get(__lowerCamelCase						,						float("inf"			)			)			)
												if bigram not in self.bpe_ranks:
															break
												A							,			A				:       Optional[int]														=    bigram
												A				:       Dict														=    []
												A				:       Optional[Any]														=    0
												while i < len(__lowerCamelCase			):
															try:
																		A				:       Dict														=    word.index(__lowerCamelCase						,						__lowerCamelCase			)
															except ValueError:
																		new_word.extend(word[i:]			)
																		break
															else:
																		new_word.extend(word[i:j]			)
																		A				:       Dict														=    j

															if word[i] == first and i < len(__lowerCamelCase			) - 1 and word[i + 1] == second:
																		new_word.append(first + second			)
																		i += 2
															else:
																		new_word.append(word[i]			)
																		i += 1
												A				:       Any														=    tuple(__lowerCamelCase			)
												A				:       Optional[int]														=    new_word
												if len(__lowerCamelCase			) == 1:
															break
												else:
															A				:       Optional[int]														=    get_pairs(__lowerCamelCase			)
									A				:       Optional[int]														=    " ".join(__lowerCamelCase			)
									if word == "\n  </w>":
												A				:       str														=    "\n</w>"
									A				:       List[Any]														=    word
									return word







						def       SCREAMING_SNAKE_CASE__		(						self   :      Tuple						,						__lowerCamelCase   :      List[str]			)   ->					List[Any]:
									A				:       Any														=    []
									if self.fix_text is None:
												# Using BERT's BasicTokenizer
												A				:       str														=    self.nlp.tokenize(__lowerCamelCase			)
												for token in text:
															split_tokens.extend(list(self.bpe(__lowerCamelCase			).split(" "			)			)			)
									else:
												# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
												A				:       List[str]														=    self.nlp(text_standardize(self.fix_text(__lowerCamelCase			)			)			)
												for token in text:
															split_tokens.extend(list(self.bpe(token.text.lower()			).split(" "			)			)			)
									return split_tokens







						def       SCREAMING_SNAKE_CASE__		(						self   :      str						,						__lowerCamelCase   :      Optional[Any]			)   ->					Optional[int]:
									return self.encoder.get(__lowerCamelCase						,						self.encoder.get(self.unk_token			)			)







						def       SCREAMING_SNAKE_CASE__		(						self   :      str						,						__lowerCamelCase   :      Any			)   ->					Optional[int]:
									return self.decoder.get(__lowerCamelCase						,						self.unk_token			)







						def       SCREAMING_SNAKE_CASE__		(						self   :      List[str]						,						__lowerCamelCase   :      str			)   ->					List[str]:
									A				:       Dict														=    "".join(__lowerCamelCase			).replace("</w>"						,						" "			).strip()
									return out_string







						def       SCREAMING_SNAKE_CASE__		(						self   :      Tuple						,						__lowerCamelCase   :      str						,						__lowerCamelCase   :      Optional[str] = None			)   ->					Tuple[str]:
									if not os.path.isdir(__lowerCamelCase			):
												logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""			)
												return
									A				:       Optional[int]														=    os.path.join(
									    __lowerCamelCase						,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]			)
									A				:       Optional[Any]														=    os.path.join(
									    __lowerCamelCase						,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]			)

									with open(__lowerCamelCase						,						"w"						,						encoding="utf-8"			) as f:
												f.write(json.dumps(self.encoder						,						indent=2						,						sort_keys=__lowerCamelCase						,						ensure_ascii=__lowerCamelCase			) + "\n"			)

									A				:       Dict														=    0
									with open(__lowerCamelCase						,						"w"						,						encoding="utf-8"			) as writer:
												writer.write("#version: 0.2\n"			)
												for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,						key=lambda __lowerCamelCase			: kv[1]			):
															if index != token_index:
																		logger.warning(
																		    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																		    " Please check that the tokenizer is not corrupted!"			)
																		A				:       Optional[int]														=    token_index
															writer.write(" ".join(__lowerCamelCase			) + "\n"			)
															index += 1

									return vocab_file, merge_file




import warnings

from ...utils import logging
from .image_processing_detr import DetrImageProcessor


__SCREAMING_SNAKE_CASE     =	logging.get_logger(__name__)

class 						lowerCamelCase_     (			_A    ):






						'''simple docstring'''







						def __init__(						self   :      Dict						,						*__lowerCamelCase   :      Tuple						,						**__lowerCamelCase   :      Dict			)   ->					None:
									warnings.warn(
									    "The class DetrFeatureExtractor is deprecated and will be removed in version 5 of Transformers."
									    " Please use DetrImageProcessor instead."						,						__lowerCamelCase						,						)
									super().__init__(*__lowerCamelCase						,						**__lowerCamelCase			)