





'''simple docstring'''


import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


_lowerCamelCase					:						Any					     =							logging.get_logger(__name__)

_lowerCamelCase					:						Dict					     =							{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

_lowerCamelCase					:						str					     =							{
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

_lowerCamelCase					:						Optional[Any]					     =							{
    'openai-gpt': 512,
}




def  __a			(  UpperCAmelCase     )   ->str:



	"""simple docstring"""







	A					=   text.strip()
	if not text:
		return []
	A					=   text.split()
	return tokens



class 				__UpperCAmelCase	(							__A    ):







		'''simple docstring'''







		def __init__(self    :   Optional[Any]   ,   _lowerCAmelCase    :   int=True   ,   _lowerCAmelCase    :   List[Any]=None   ,   _lowerCAmelCase    :   Dict=True   ,   _lowerCAmelCase    :   int=None   ,   _lowerCAmelCase    :   Optional[Any]=True   ,   ):
			if never_split is None:
				A					=   []
			A					=   do_lower_case
			A					=   set(lowercase_   )
			A					=   tokenize_chinese_chars
			A					=   strip_accents
			A					=   do_split_on_punc







		def 							A  (self    :   Dict   ,   _lowerCAmelCase    :   Union[str, Any]   ,   _lowerCAmelCase    :   List[Any]=None   ):
			A					=   self.never_split.union(set(lowercase_   )   ) if never_split else self.never_split
			A					=   self._clean_text(lowercase_   )

			# This was added on November 1st, 2018 for the multilingual and Chinese
			# models. This is also applied to the English models now, but it doesn't
			# matter since the English models were not trained on any Chinese data
			# and generally don't have any Chinese data in them (there are Chinese
			# characters in the vocabulary because Wikipedia does have some Chinese
			# words in the English Wikipedia.).
			if self.tokenize_chinese_chars:
				A					=   self._tokenize_chinese_chars(lowercase_   )
			# prevents treating the same character with different unicode codepoints as different characters
			A					=   unicodedata.normalize("""NFC"""   ,   lowercase_   )
			A					=   whitespace_tokenize(lowercase_   )
			A					=   []
			for token in orig_tokens:
				if token not in never_split:
					if self.do_lower_case:
						A					=   token.lower()
						if self.strip_accents is not False:
							A					=   self._run_strip_accents(lowercase_   )
					elif self.strip_accents:
						A					=   self._run_strip_accents(lowercase_   )
				split_tokens.extend(self._run_split_on_punc(lowercase_   ,   lowercase_   )   )

			A					=   whitespace_tokenize(""" """.join(lowercase_   )   )
			return output_tokens







		def 							A  (self    :   Optional[int]   ,   _lowerCAmelCase    :   Dict   ):
			A					=   unicodedata.normalize("""NFD"""   ,   lowercase_   )
			A					=   []
			for char in text:
				A					=   unicodedata.category(lowercase_   )
				if cat == "Mn":
					continue
				output.append(lowercase_   )
			return "".join(lowercase_   )







		def 							A  (self    :   List[str]   ,   _lowerCAmelCase    :   str   ,   _lowerCAmelCase    :   List[Any]=None   ):
			if not self.do_split_on_punc or (never_split is not None and text in never_split):
				return [text]
			A					=   list(lowercase_   )
			A					=   0
			A					=   True
			A					=   []
			while i < len(lowercase_   ):
				A					=   chars[i]
				if _is_punctuation(lowercase_   ):
					output.append([char]   )
					A					=   True
				else:
					if start_new_word:
						output.append([]   )
					A					=   False
					output[-1].append(lowercase_   )
				i += 1

			return ["".join(lowercase_   ) for x in output]







		def 							A  (self    :   Any   ,   _lowerCAmelCase    :   List[Any]   ):
			A					=   []
			for char in text:
				A					=   ord(lowercase_   )
				if self._is_chinese_char(lowercase_   ):
					output.append(""" """   )
					output.append(lowercase_   )
					output.append(""" """   )
				else:
					output.append(lowercase_   )
			return "".join(lowercase_   )







		def 							A  (self    :   Union[str, Any]   ,   _lowerCAmelCase    :   List[str]   ):
			# This defines a "chinese character" as anything in the CJK Unicode block:
			#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
			#
			# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
			# despite its name. The modern Korean Hangul alphabet is a different block,
			# as is Japanese Hiragana and Katakana. Those alphabets are used to write
			# space-separated words, so they are not treated specially and handled
			# like the all of the other languages.
			if (
			    (cp >= 0X4_e00 and cp <= 0X9_fff)
			    or (cp >= 0X3_400 and cp <= 0X4_dbf)  #
			    or (cp >= 0X20_000 and cp <= 0X2a_6df)  #
			    or (cp >= 0X2a_700 and cp <= 0X2b_73f)  #
			    or (cp >= 0X2b_740 and cp <= 0X2b_81f)  #
			    or (cp >= 0X2b_820 and cp <= 0X2c_eaf)  #
			    or (cp >= 0Xf_900 and cp <= 0Xf_aff)
			    or (cp >= 0X2f_800 and cp <= 0X2f_a1f)  #
			):  #
				return True

			return False







		def 							A  (self    :   str   ,   _lowerCAmelCase    :   List[Any]   ):
			A					=   []
			for char in text:
				A					=   ord(lowercase_   )
				if cp == 0 or cp == 0Xf_ffd or _is_control(lowercase_   ):
					continue
				if _is_whitespace(lowercase_   ):
					output.append(""" """   )
				else:
					output.append(lowercase_   )
			return "".join(lowercase_   )






def  __a			(  UpperCAmelCase     )   ->int:



	"""simple docstring"""







	A					=   set()
	A					=   word[0]
	for char in word[1:]:
		pairs.add((prev_char, char)     )
		A					=   char
	return pairs




def  __a			(  UpperCAmelCase     )   ->Tuple:



	"""simple docstring"""







	A					=   text.replace("""—"""							,      """-"""     )
	A					=   text.replace("""–"""							,      """-"""     )
	A					=   text.replace("""―"""							,      """-"""     )
	A					=   text.replace("""…"""							,      """..."""     )
	A					=   text.replace("""´"""							,      """'"""     )
	A					=   re.sub(R"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"""							,      R""" \1 """							,      _lowerCamelCase     )
	A					=   re.sub(R"""\s*\n\s*"""							,      """ \n """							,      _lowerCamelCase     )
	A					=   re.sub(R"""[^\S\n]+"""							,      """ """							,      _lowerCamelCase     )
	return text.strip()



class 				__UpperCAmelCase	(							__A    ):







		'''simple docstring'''







		__lowerCAmelCase								=     VOCAB_FILES_NAMES
		__lowerCAmelCase								=     PRETRAINED_VOCAB_FILES_MAP
		__lowerCAmelCase								=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		__lowerCAmelCase								=     ["input_ids", "attention_mask"]







		def __init__(self    :   Optional[int]   ,   _lowerCAmelCase    :   List[str]   ,   _lowerCAmelCase    :   Dict   ,   _lowerCAmelCase    :   Optional[int]="<unk>"   ,   **_lowerCAmelCase    :   Dict   ):
			super().__init__(unk_token=lowercase_   ,   **lowercase_   )

			try:
				import ftfy
				from spacy.lang.en import English

				A					=   English()
				A					=   _nlp.tokenizer
				A					=   ftfy.fix_text
			except ImportError:
				logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""   )
				A					=   BasicTokenizer(do_lower_case=lowercase_   )
				A					=   None

			with open(lowercase_   ,   encoding="""utf-8"""   ) as vocab_handle:
				A					=   json.load(lowercase_   )
			A					=   {v: k for k, v in self.encoder.items()}
			with open(lowercase_   ,   encoding="""utf-8"""   ) as merges_handle:
				A					=   merges_handle.read().split("""\n"""   )[1:-1]
			A					=   [tuple(merge.split()   ) for merge in merges]
			A					=   dict(zip(lowercase_   ,   range(len(lowercase_   )   )   )   )
			A					=   {}







		@property
		def 							A  (self    :   Union[str, Any]   ):
			return True







		@property
		def 							A  (self    :   str   ):
			return len(self.encoder   )







		def 							A  (self    :   List[str]   ):
			return dict(self.encoder   ,   **self.added_tokens_encoder   )







		def 							A  (self    :   Optional[Any]   ,   _lowerCAmelCase    :   Dict   ):
			A					=   tuple(token[:-1]   ) + (token[-1] + "</w>",)
			if token in self.cache:
				return self.cache[token]
			A					=   get_pairs(lowercase_   )

			if not pairs:
				return token + "</w>"

			while True:
				A					=   min(lowercase_   ,   key=lambda _lowerCAmelCase   : self.bpe_ranks.get(lowercase_   ,   float("""inf"""   )   )   )
				if bigram not in self.bpe_ranks:
					break
				A					=   bigram
				A					=   []
				A					=   0
				while i < len(lowercase_   ):
					try:
						A					=   word.index(lowercase_   ,   lowercase_   )
					except ValueError:
						new_word.extend(word[i:]   )
						break
					else:
						new_word.extend(word[i:j]   )
						A					=   j

					if word[i] == first and i < len(lowercase_   ) - 1 and word[i + 1] == second:
						new_word.append(first + second   )
						i += 2
					else:
						new_word.append(word[i]   )
						i += 1
				A					=   tuple(lowercase_   )
				A					=   new_word
				if len(lowercase_   ) == 1:
					break
				else:
					A					=   get_pairs(lowercase_   )
			A					=   " ".join(lowercase_   )
			if word == "\n  </w>":
				A					=   "\n</w>"
			A					=   word
			return word







		def 							A  (self    :   List[Any]   ,   _lowerCAmelCase    :   List[Any]   ):
			A					=   []
			if self.fix_text is None:
				# Using BERT's BasicTokenizer
				A					=   self.nlp.tokenize(lowercase_   )
				for token in text:
					split_tokens.extend(list(self.bpe(lowercase_   ).split(""" """   )   )   )
			else:
				# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
				A					=   self.nlp(text_standardize(self.fix_text(lowercase_   )   )   )
				for token in text:
					split_tokens.extend(list(self.bpe(token.text.lower()   ).split(""" """   )   )   )
			return split_tokens







		def 							A  (self    :   Dict   ,   _lowerCAmelCase    :   Any   ):
			return self.encoder.get(lowercase_   ,   self.encoder.get(self.unk_token   )   )







		def 							A  (self    :   Optional[int]   ,   _lowerCAmelCase    :   Any   ):
			return self.decoder.get(lowercase_   ,   self.unk_token   )







		def 							A  (self    :   str   ,   _lowerCAmelCase    :   Any   ):
			A					=   "".join(lowercase_   ).replace("""</w>"""   ,   """ """   ).strip()
			return out_string







		def 							A  (self    :   Dict   ,   _lowerCAmelCase    :   str   ,   _lowerCAmelCase    :   Optional[str] = None   ):
			if not os.path.isdir(lowercase_   ):
				logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
				return
			A					=   os.path.join(
			    lowercase_   ,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )
			A					=   os.path.join(
			    lowercase_   ,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]   )

			with open(lowercase_   ,   """w"""   ,   encoding="""utf-8"""   ) as f:
				f.write(json.dumps(self.encoder   ,   indent=2   ,   sort_keys=lowercase_   ,   ensure_ascii=lowercase_   ) + """\n"""   )

			A					=   0
			with open(lowercase_   ,   """w"""   ,   encoding="""utf-8"""   ) as writer:
				writer.write("""#version: 0.2\n"""   )
				for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,   key=lambda _lowerCAmelCase   : kv[1]   ):
					if index != token_index:
						logger.warning(
						    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
						    """ Please check that the tokenizer is not corrupted!"""   )
						A					=   token_index
					writer.write(""" """.join(lowercase_   ) + """\n"""   )
					index += 1

			return vocab_file, merge_file


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


UpperCamelCase    				=		{'''configuration_dinat''': ['''DINAT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''DinatConfig''']}


try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      UpperCamelCase    				=		[
          '''DINAT_PRETRAINED_MODEL_ARCHIVE_LIST''',
          '''DinatForImageClassification''',
          '''DinatModel''',
          '''DinatPreTrainedModel''',
          '''DinatBackbone''',
      ]

if TYPE_CHECKING:
      from .configuration_dinat import DINAT_PRETRAINED_CONFIG_ARCHIVE_MAP, DinatConfig

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_dinat import (
                DINAT_PRETRAINED_MODEL_ARCHIVE_LIST,
                DinatBackbone,
                DinatForImageClassification,
                DinatModel,
                DinatPreTrainedModel,
            )

else:
      import sys

      UpperCamelCase    				=		_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)



