


"""simple docstring"""






import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation

__snake_case												=							logging.get_logger(__name__)

__snake_case												=							{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

__snake_case												=							{
    'vocab_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/vocab.json',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/vocab.json',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/vocab.json',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/vocab.json',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/vocab.json',
    },
    'merges_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/merges.txt',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/merges.txt',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/merges.txt',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/merges.txt',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/merges.txt',
    },
}

__snake_case												=							{
    'gpt2': 1024,
    'gpt2-medium': 1024,
    'gpt2-large': 1024,
    'gpt2-xl': 1024,
    'distilgpt2': 1024,
}
@lru_cache()
def    A_ ( ):






     """simple docstring"""

     _a     						=					(
         list(range(ord('''!'''							),  ord('''~'''							) + 1							)							) + list(range(ord('''¡'''							),  ord('''¬'''							) + 1							)							) + list(range(ord('''®'''							),  ord('''ÿ'''							) + 1							)							)
     )
     _a     						=					bs[:]
     _a     						=					0
     for b in range(2**8							):
          if b not in bs:
               bs.append(_A							)
               cs.append(2**8 + n							)
               n += 1
     _a     						=					[chr(_A							) for n in cs]
     return dict(zip(_A,  _A							)							)
def    A_ ( _lowerCAmelCase  :       Optional[int]							):






     """simple docstring"""

     _a     						=					set()
     _a     						=					word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)							)
          _a     						=					char
     return pairs


class      __lowerCamelCase	(       lowerCAmelCase__					):

     '''simple docstring'''
     A_							:						Any     =		VOCAB_FILES_NAMES
     A_							:						Any     =		PRETRAINED_VOCAB_FILES_MAP
     A_							:						Optional[int]     =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     A_							:						List[str]     =		["input_ids", "attention_mask"]


     def __init__(   self     ,					__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase="replace"     ,					__UpperCAmelCase="<|endoftext|>"     ,					__UpperCAmelCase="<|endoftext|>"     ,					__UpperCAmelCase="<|endoftext|>"     ,					__UpperCAmelCase=None     ,					__UpperCAmelCase=False     ,					__UpperCAmelCase=False     ,					**__UpperCAmelCase     ,					)							->				Optional[Any]:
          _a     						=					AddedToken(_SCREAMING_SNAKE_CASE     ,					lstrip=_SCREAMING_SNAKE_CASE     ,					rstrip=_SCREAMING_SNAKE_CASE				) if isinstance(_SCREAMING_SNAKE_CASE     ,					_SCREAMING_SNAKE_CASE				) else bos_token
          _a     						=					AddedToken(_SCREAMING_SNAKE_CASE     ,					lstrip=_SCREAMING_SNAKE_CASE     ,					rstrip=_SCREAMING_SNAKE_CASE				) if isinstance(_SCREAMING_SNAKE_CASE     ,					_SCREAMING_SNAKE_CASE				) else eos_token
          _a     						=					AddedToken(_SCREAMING_SNAKE_CASE     ,					lstrip=_SCREAMING_SNAKE_CASE     ,					rstrip=_SCREAMING_SNAKE_CASE				) if isinstance(_SCREAMING_SNAKE_CASE     ,					_SCREAMING_SNAKE_CASE				) else unk_token
          _a     						=					AddedToken(_SCREAMING_SNAKE_CASE     ,					lstrip=_SCREAMING_SNAKE_CASE     ,					rstrip=_SCREAMING_SNAKE_CASE				) if isinstance(_SCREAMING_SNAKE_CASE     ,					_SCREAMING_SNAKE_CASE				) else pad_token
          super().__init__(
              errors=_SCREAMING_SNAKE_CASE     ,					unk_token=_SCREAMING_SNAKE_CASE     ,					bos_token=_SCREAMING_SNAKE_CASE     ,					eos_token=_SCREAMING_SNAKE_CASE     ,					pad_token=_SCREAMING_SNAKE_CASE     ,					add_prefix_space=_SCREAMING_SNAKE_CASE     ,					add_bos_token=_SCREAMING_SNAKE_CASE     ,					**_SCREAMING_SNAKE_CASE     ,					)
          _a     						=					add_bos_token

          with open(_SCREAMING_SNAKE_CASE     ,					encoding='''utf-8'''				) as vocab_handle:
               _a     						=					json.load(_SCREAMING_SNAKE_CASE				)
          _a     						=					{v: k for k, v in self.encoder.items()}
          _a     						=					errors  # how to handle errors in decoding
          _a     						=					bytes_to_unicode()
          _a     						=					{v: k for k, v in self.byte_encoder.items()}
          with open(_SCREAMING_SNAKE_CASE     ,					encoding='''utf-8'''				) as merges_handle:
               _a     						=					merges_handle.read().split('''\n'''				)[1:-1]
          _a     						=					[tuple(merge.split()				) for merge in bpe_merges]
          _a     						=					dict(zip(_SCREAMING_SNAKE_CASE     ,					range(len(_SCREAMING_SNAKE_CASE				)				)				)				)
          _a     						=					{}
          _a     						=					add_prefix_space

          # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
          _a     						=					re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''				)


     @property
     def 				_UpperCAmelCase   (   self				)							->				str:
          return len(self.encoder				)


     def 				_UpperCAmelCase   (   self				)							->				Optional[Any]:
          return dict(self.encoder     ,					**self.added_tokens_encoder				)


     def 				_UpperCAmelCase   (   self     ,					__UpperCAmelCase				)							->				int:
          if token in self.cache:
               return self.cache[token]
          _a     						=					tuple(_SCREAMING_SNAKE_CASE				)
          _a     						=					get_pairs(_SCREAMING_SNAKE_CASE				)

          if not pairs:
               return token

          while True:
               _a     						=					min(_SCREAMING_SNAKE_CASE     ,					key=lambda __UpperCAmelCase				: self.bpe_ranks.get(_SCREAMING_SNAKE_CASE     ,					float('''inf'''				)				)				)
               if bigram not in self.bpe_ranks:
                    break
               _a						,   _a     						=					bigram
               _a     						=					[]
               _a     						=					0
               while i < len(_SCREAMING_SNAKE_CASE				):
                    try:
                         _a     						=					word.index(_SCREAMING_SNAKE_CASE     ,					_SCREAMING_SNAKE_CASE				)
                    except ValueError:
                         new_word.extend(word[i:]				)
                         break
                    else:
                         new_word.extend(word[i:j]				)
                         _a     						=					j

                    if word[i] == first and i < len(_SCREAMING_SNAKE_CASE				) - 1 and word[i + 1] == second:
                         new_word.append(first + second				)
                         i += 2
                    else:
                         new_word.append(word[i]				)
                         i += 1
               _a     						=					tuple(_SCREAMING_SNAKE_CASE				)
               _a     						=					new_word
               if len(_SCREAMING_SNAKE_CASE				) == 1:
                    break
               else:
                    _a     						=					get_pairs(_SCREAMING_SNAKE_CASE				)
          _a     						=					''' '''.join(_SCREAMING_SNAKE_CASE				)
          _a     						=					word
          return word


     def 				_UpperCAmelCase   (   self     ,					__UpperCAmelCase     ,					__UpperCAmelCase=None				)							->				Any:
          if self.add_bos_token:
               _a     						=					[self.bos_token_id]
          else:
               _a     						=					[]

          _a     						=					bos_token_ids + token_ids_a

          if token_ids_a is None:
               return output

          return output + bos_token_ids + token_ids_a


     def 				_UpperCAmelCase   (   self     ,					__UpperCAmelCase     ,					__UpperCAmelCase = None     ,					__UpperCAmelCase = False				)							->				List[int]:
          if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=_SCREAMING_SNAKE_CASE     ,					token_ids_a=_SCREAMING_SNAKE_CASE     ,					already_has_special_tokens=_SCREAMING_SNAKE_CASE				)

          if not self.add_bos_token:
               return super().get_special_tokens_mask(
                   token_ids_a=_SCREAMING_SNAKE_CASE     ,					token_ids_a=_SCREAMING_SNAKE_CASE     ,					already_has_special_tokens=_SCREAMING_SNAKE_CASE				)

          if token_ids_a is None:
               return [1] + ([0] * len(_SCREAMING_SNAKE_CASE				))
          return [1] + ([0] * len(_SCREAMING_SNAKE_CASE				)) + [1] + ([0] * len(_SCREAMING_SNAKE_CASE				))


     def 				_UpperCAmelCase   (   self     ,					__UpperCAmelCase				)							->				Any:
          _a     						=					[]
          for token in re.findall(self.pat     ,					_SCREAMING_SNAKE_CASE				):
               _a     						=					''''''.join(
                   self.byte_encoder[b] for b in token.encode('''utf-8'''				)				)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
               bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_SCREAMING_SNAKE_CASE				).split(''' '''				)				)
          return bpe_tokens


     def 				_UpperCAmelCase   (   self     ,					__UpperCAmelCase				)							->				List[str]:
          return self.encoder.get(_SCREAMING_SNAKE_CASE     ,					self.encoder.get(self.unk_token				)				)


     def 				_UpperCAmelCase   (   self     ,					__UpperCAmelCase				)							->				int:
          return self.decoder.get(_SCREAMING_SNAKE_CASE				)


     def 				_UpperCAmelCase   (   self     ,					__UpperCAmelCase				)							->				int:
          _a     						=					''''''.join(_SCREAMING_SNAKE_CASE				)
          _a     						=					bytearray([self.byte_decoder[c] for c in text]				).decode('''utf-8'''     ,					errors=self.errors				)
          return text


     def 				_UpperCAmelCase   (   self     ,					__UpperCAmelCase     ,					__UpperCAmelCase = None				)							->				Tuple[str]:
          if not os.path.isdir(_SCREAMING_SNAKE_CASE				):
               logger.error(F'Vocabulary path ({save_directory}) should be a directory'				)
               return
          _a     						=					os.path.join(
              _SCREAMING_SNAKE_CASE     ,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)
          _a     						=					os.path.join(
              _SCREAMING_SNAKE_CASE     ,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']				)

          with open(_SCREAMING_SNAKE_CASE     ,					'''w'''     ,					encoding='''utf-8'''				) as f:
               f.write(json.dumps(self.encoder     ,					indent=2     ,					sort_keys=_SCREAMING_SNAKE_CASE     ,					ensure_ascii=_SCREAMING_SNAKE_CASE				) + '''\n'''				)

          _a     						=					0
          with open(_SCREAMING_SNAKE_CASE     ,					'''w'''     ,					encoding='''utf-8'''				) as writer:
               writer.write('''#version: 0.2\n'''				)
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,					key=lambda __UpperCAmelCase				: kv[1]				):
                    if index != token_index:
                         logger.warning(
                             F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                             ''' Please check that the tokenizer is not corrupted!'''				)
                         _a     						=					token_index
                    writer.write(''' '''.join(_SCREAMING_SNAKE_CASE				) + '''\n'''				)
                    index += 1

          return vocab_file, merge_file


     def 				_UpperCAmelCase   (   self     ,					__UpperCAmelCase     ,					__UpperCAmelCase=False     ,					**__UpperCAmelCase				)							->				Optional[Any]:
          _a     						=					kwargs.pop('''add_prefix_space'''     ,					self.add_prefix_space				)
          if is_split_into_words or add_prefix_space:
               _a     						=					''' ''' + text
          return (text, kwargs)





     def 				_UpperCAmelCase   (   self     ,					__UpperCAmelCase				)							->				List[int]:
          _a     						=					[]
          for is_user, text in conversation.iter_texts():
               input_ids.extend(self.encode(_SCREAMING_SNAKE_CASE     ,					add_special_tokens=_SCREAMING_SNAKE_CASE				) + [self.eos_token_id]				)
          if len(_SCREAMING_SNAKE_CASE				) > self.model_max_length:
               _a     						=					input_ids[-self.model_max_length :]
          return input_ids


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
    is_vision_available,
)


__A	:   Optional[int]	=					{
    'configuration_mobilevitv2': [
        'MOBILEVITV2_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'MobileViTV2Config',
        'MobileViTV2OnnxConfig',
    ],
}


try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__A	:   Tuple	=					[
	    'MOBILEVITV2_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'MobileViTV2ForImageClassification',
	    'MobileViTV2ForSemanticSegmentation',
	    'MobileViTV2Model',
	    'MobileViTV2PreTrainedModel',
	]

if TYPE_CHECKING:
	from .configuration_mobilevitva import (
	    MOBILEVITV2_PRETRAINED_CONFIG_ARCHIVE_MAP,
	    MobileViTVaConfig,
	    MobileViTVaOnnxConfig,
	)

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_mobilevitva import (
		    MOBILEVITV2_PRETRAINED_MODEL_ARCHIVE_LIST,
		    MobileViTVaForImageClassification,
		    MobileViTVaForSemanticSegmentation,
		    MobileViTVaModel,
		    MobileViTVaPreTrainedModel,
		)

else:
	import sys

	__A	:   List[str]	=					_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)
