



"""simple docstring"""







import os
from shutil import copyfile
from typing import TYPE_CHECKING, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging
from ...utils.versions import require_version


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation

require_version("""tokenizers>=0.13.3""")

if is_sentencepiece_available():
     from .tokenization_llama import LlamaTokenizer
else:
     lowercase__						:							int        =      None

lowercase__						:							Optional[Any]        =      logging.get_logger(__name__)
lowercase__						:							Union[str, Any]        =      {"""vocab_file""": """tokenizer.model""", """tokenizer_file""": """tokenizer.json"""}

lowercase__       ,	lowercase__						:							Any        =      """[INST]""", """[/INST]"""
lowercase__       ,	lowercase__						:							Optional[int]        =      """<<SYS>>\n""", """\n<</SYS>>\n\n"""

# fmt: off
lowercase__						:							Optional[Any]        =      """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\
answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\
that your responses are socially unbiased and positive in nature.

If a question does not make any sense, or is not factually coherent, explain why instead of answering something not\
correct. If you don\'t know the answer to a question, please don\'t share false information."""







class  UpperCamelCase__  (      __lowerCAmelCase    ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE    							=	VOCAB_FILES_NAMES
      _SCREAMING_SNAKE_CASE    							=	LlamaTokenizer
      _SCREAMING_SNAKE_CASE    							=	"""left"""
      _SCREAMING_SNAKE_CASE    							=	["""input_ids""", """attention_mask"""]



      def __init__( self							: Any       ,     SCREAMING_SNAKE_CASE_							: Tuple=None       ,     SCREAMING_SNAKE_CASE_							: Any=None       ,     SCREAMING_SNAKE_CASE_							: List[str]=False       ,     SCREAMING_SNAKE_CASE_							: List[Any]="<unk>"       ,     SCREAMING_SNAKE_CASE_							: Optional[int]="<s>"       ,     SCREAMING_SNAKE_CASE_							: Tuple="</s>"       ,     SCREAMING_SNAKE_CASE_							: Any=True       ,     SCREAMING_SNAKE_CASE_							: Any=False       ,     **SCREAMING_SNAKE_CASE_							: Tuple       ,     ):
        super().__init__(
            vocab_file=__UpperCamelCase       ,     tokenizer_file=__UpperCamelCase       ,     clean_up_tokenization_spaces=__UpperCamelCase       ,     unk_token=__UpperCamelCase       ,     bos_token=__UpperCamelCase       ,     eos_token=__UpperCamelCase       ,     **__UpperCamelCase       ,     )
        lowerCAmelCase_  :							Optional[int]								=					add_bos_token
        lowerCAmelCase_  :							int								=					add_eos_token
        self.update_post_processor()

        lowerCAmelCase_  :							Tuple								=					vocab_file
        lowerCAmelCase_  :							int								=					False if not self.vocab_file else True



      def 	SCREAMING_SNAKE_CASE__					( self							: List[Any]			):
        lowerCAmelCase_  :							List[Any]								=					self.bos_token
        lowerCAmelCase_  :							Optional[int]								=					self.bos_token_id

        lowerCAmelCase_  :							Union[str, Any]								=					self.eos_token
        lowerCAmelCase_  :							Optional[Any]								=					self.eos_token_id

        lowerCAmelCase_  :							int								=					F"{(bos+':0 ') * self.add_bos_token}$A:0{(' '+eos+':0') * self.add_eos_token}"
        lowerCAmelCase_  :							List[str]								=					F"{single}{(' '+bos+':1') * self.add_bos_token} $B:1{(' '+eos+':1') * self.add_eos_token}"

        lowerCAmelCase_  :							int								=					[]
        if self.add_bos_token:
          special_tokens.append((bos, bos_token_id)			)
        if self.add_eos_token:
          special_tokens.append((eos, eos_token_id)			)
        lowerCAmelCase_  :							Dict								=					processors.TemplateProcessing(
            single=__UpperCamelCase       ,     pair=__UpperCamelCase       ,     special_tokens=__UpperCamelCase			)



      @property
      def 	SCREAMING_SNAKE_CASE__					( self							: int			):
        return self._add_eos_token



      @property
      def 	SCREAMING_SNAKE_CASE__					( self							: str			):
        return self._add_bos_token



      @add_eos_token.setter
      def 	SCREAMING_SNAKE_CASE__					( self							: Union[str, Any]       ,     SCREAMING_SNAKE_CASE_							: Any			):
        lowerCAmelCase_  :							Dict								=					value
        self.update_post_processor()



      @add_bos_token.setter
      def 	SCREAMING_SNAKE_CASE__					( self							: int       ,     SCREAMING_SNAKE_CASE_							: Any			):
        lowerCAmelCase_  :							int								=					value
        self.update_post_processor()



      def 	SCREAMING_SNAKE_CASE__					( self							: Any       ,     SCREAMING_SNAKE_CASE_							: str       ,     SCREAMING_SNAKE_CASE_							: Optional[str] = None			):






        if not self.can_save_slow_tokenizer:
          raise ValueError(
              'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
              'tokenizer.'			)

        if not os.path.isdir(__UpperCamelCase			):
          logger.error(F"Vocabulary path ({save_directory}) should be a directory"			)
          return
        lowerCAmelCase_  :							Any								=					os.path.join(
            __UpperCamelCase       ,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']			)

        if os.path.abspath(self.vocab_file			) != os.path.abspath(__UpperCamelCase			):
          copyfile(self.vocab_file       ,     __UpperCamelCase			)

        return (out_vocab_file,)





      def 	SCREAMING_SNAKE_CASE__					( self							: Optional[int]       ,     SCREAMING_SNAKE_CASE_							: "Conversation"			):
        lowerCAmelCase_  :							Optional[Any]								=					list(conversation.iter_texts()			)
        if not all([is_user for is_user, msg in dialogue[::2]]			) or not all(
            [not is_user for is_user, msg in dialogue[1::2]]			):
          raise ValueError(
              'The model only supports \'user\' and \'assistant\' roles, starting with user and alternating (u/a/u/a/u...)'			)

        lowerCAmelCase_  :							int								=					[]
        if len(conversation.past_user_inputs			) > 0:
          if not conversation.past_user_inputs[0].startswith(__UpperCamelCase			) or E_SYS not in conversation.past_user_inputs[0]:
            lowerCAmelCase_  :							Dict								=					(
                B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
            )
        elif not dialogue[0][1].startswith(__UpperCamelCase			) or E_SYS not in dialogue[0][1]:
          lowerCAmelCase_  :							str								=					(dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

        dialog_tokens += sum(
            [
                [self.bos_token_id]
                + self.encode(
                    F"{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} "       ,     add_special_tokens=__UpperCamelCase			)
                + [self.eos_token_id]
                for prompt, answer in zip(dialogue[::2]       ,     dialogue[1::2]			)
            ]       ,     []       ,     )
        if not (dialogue[-1][0]):
          raise ValueError(F"Last message must be from user, got {dialogue[-1]['role']}"			)
        dialog_tokens += [self.bos_token_id] + self.encode(
            F"{B_INST} {(dialogue[-1][1]).strip()} {E_INST}"       ,     add_special_tokens=__UpperCamelCase			)
        return dialog_tokens











'''simple docstring'''

import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


a__         =     logging.get_logger(__name__)

a__         =     {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

a__         =     {
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

a__         =     {
    '''openai-gpt''': 512,
}




def    snake_case__     (  a	)			->       Tuple:





			'''simple docstring'''






			snake_case__          =						text.strip()
			if not text:
						return []
			snake_case__          =						text.split()
			return tokens





class __magic_name__(						__lowerCAmelCase      ):






						def __init__(   self						:						Any					,   __UpperCamelCase						:						str=True					,   __UpperCamelCase						:						Dict=None					,   __UpperCamelCase						:						Union[str, Any]=True					,   __UpperCamelCase						:						List[str]=None					,   __UpperCamelCase						:						Optional[int]=True					,   ):



									'''simple docstring'''






									if never_split is None:
												snake_case__          =						[]
									snake_case__          =						do_lower_case
									snake_case__          =						set(__UpperCamelCase       )
									snake_case__          =						tokenize_chinese_chars
									snake_case__          =						strip_accents
									snake_case__          =						do_split_on_punc






						def       __lowerCAmelCase(   self						:						str					,   __UpperCamelCase						:						Optional[int]					,   __UpperCamelCase						:						Any=None       ):



									'''simple docstring'''






									snake_case__          =						self.never_split.union(set(__UpperCamelCase       )       ) if never_split else self.never_split
									snake_case__          =						self._clean_text(__UpperCamelCase       )

									# This was added on November 1st, 2018 for the multilingual and Chinese
									# models. This is also applied to the English models now, but it doesn't
									# matter since the English models were not trained on any Chinese data
									# and generally don't have any Chinese data in them (there are Chinese
									# characters in the vocabulary because Wikipedia does have some Chinese
									# words in the English Wikipedia.).
									if self.tokenize_chinese_chars:
												snake_case__          =						self._tokenize_chinese_chars(__UpperCamelCase       )
									# prevents treating the same character with different unicode codepoints as different characters
									snake_case__          =						unicodedata.normalize("""NFC"""					,   __UpperCamelCase       )
									snake_case__          =						whitespace_tokenize(__UpperCamelCase       )
									snake_case__          =						[]
									for token in orig_tokens:
												if token not in never_split:
															if self.do_lower_case:
																		snake_case__          =						token.lower()
																		if self.strip_accents is not False:
																					snake_case__          =						self._run_strip_accents(__UpperCamelCase       )
															elif self.strip_accents:
																		snake_case__          =						self._run_strip_accents(__UpperCamelCase       )
												split_tokens.extend(self._run_split_on_punc(__UpperCamelCase					,   __UpperCamelCase       )       )

									snake_case__          =						whitespace_tokenize(""" """.join(__UpperCamelCase       )       )
									return output_tokens






						def       __lowerCAmelCase(   self						:						Optional[Any]					,   __UpperCamelCase						:						Tuple       ):



									'''simple docstring'''






									snake_case__          =						unicodedata.normalize("""NFD"""					,   __UpperCamelCase       )
									snake_case__          =						[]
									for char in text:
												snake_case__          =						unicodedata.category(__UpperCamelCase       )
												if cat == "Mn":
															continue
												output.append(__UpperCamelCase       )
									return "".join(__UpperCamelCase       )






						def       __lowerCAmelCase(   self						:						Optional[Any]					,   __UpperCamelCase						:						Any					,   __UpperCamelCase						:						int=None       ):



									'''simple docstring'''






									if not self.do_split_on_punc or (never_split is not None and text in never_split):
												return [text]
									snake_case__          =						list(__UpperCamelCase       )
									snake_case__          =						0
									snake_case__          =						True
									snake_case__          =						[]
									while i < len(__UpperCamelCase       ):
												snake_case__          =						chars[i]
												if _is_punctuation(__UpperCamelCase       ):
															output.append([char]       )
															snake_case__          =						True
												else:
															if start_new_word:
																		output.append([]       )
															snake_case__          =						False
															output[-1].append(__UpperCamelCase       )
												i += 1

									return ["".join(__UpperCamelCase       ) for x in output]






						def       __lowerCAmelCase(   self						:						Tuple					,   __UpperCamelCase						:						Optional[Any]       ):



									'''simple docstring'''






									snake_case__          =						[]
									for char in text:
												snake_case__          =						ord(__UpperCamelCase       )
												if self._is_chinese_char(__UpperCamelCase       ):
															output.append(""" """       )
															output.append(__UpperCamelCase       )
															output.append(""" """       )
												else:
															output.append(__UpperCamelCase       )
									return "".join(__UpperCamelCase       )






						def       __lowerCAmelCase(   self						:						Any					,   __UpperCamelCase						:						str       ):



									'''simple docstring'''






									if (
									    (cp >= 0X4_e00 and cp <= 0X9_fff)
									    or (cp >= 0X3_400 and cp <= 0X4_dbf)  #
									    or (cp >= 0X20_000 and cp <= 0X2a_6df)  #
									    or (cp >= 0X2a_700 and cp <= 0X2b_73f)  #
									    or (cp >= 0X2b_740 and cp <= 0X2b_81f)  #
									    or (cp >= 0X2b_820 and cp <= 0X2c_eaf)  #
									    or (cp >= 0Xf_900 and cp <= 0Xf_aff)
									    or (cp >= 0X2f_800 and cp <= 0X2f_a1f)  #
									):  #
												return True

									return False







						def       __lowerCAmelCase(   self						:						str					,   __UpperCamelCase						:						Any       ):



									'''simple docstring'''






									snake_case__          =						[]
									for char in text:
												snake_case__          =						ord(__UpperCamelCase       )
												if cp == 0 or cp == 0Xf_ffd or _is_control(__UpperCamelCase       ):
															continue
												if _is_whitespace(__UpperCamelCase       ):
															output.append(""" """       )
												else:
															output.append(__UpperCamelCase       )
									return "".join(__UpperCamelCase       )







def    snake_case__     (  a	)			->       int:





			'''simple docstring'''






			snake_case__          =						set()
			snake_case__          =						word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)	)
						snake_case__          =						char
			return pairs




def    snake_case__     (  a	)			->       Union[str, Any]:





			'''simple docstring'''






			snake_case__          =						text.replace("""—"""       ,      """-"""	)
			snake_case__          =						text.replace("""–"""       ,      """-"""	)
			snake_case__          =						text.replace("""―"""       ,      """-"""	)
			snake_case__          =						text.replace("""…"""       ,      """..."""	)
			snake_case__          =						text.replace("""´"""       ,      """'"""	)
			snake_case__          =						re.sub(R"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"""       ,      R""" \1 """       ,      a	)
			snake_case__          =						re.sub(R"""\s*\n\s*"""       ,      """ \n """       ,      a	)
			snake_case__          =						re.sub(R"""[^\S\n]+"""       ,      """ """       ,      a	)
			return text.strip()





class __magic_name__(						__lowerCAmelCase      ):
						UpperCAmelCase_		:					str    =  VOCAB_FILES_NAMES
						UpperCAmelCase_		:					Dict    =  PRETRAINED_VOCAB_FILES_MAP
						UpperCAmelCase_		:					Optional[Any]    =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCAmelCase_		:					List[str]    =  ["""input_ids""", """attention_mask"""]






						def __init__(   self						:						int					,   __UpperCamelCase						:						Optional[int]					,   __UpperCamelCase						:						str					,   __UpperCamelCase						:						Dict="<unk>"					,   **__UpperCamelCase						:						Optional[Any]       ):



									'''simple docstring'''






									super().__init__(unk_token=__UpperCamelCase					,   **__UpperCamelCase       )

									try:
												import ftfy
												from spacy.lang.en import English

												snake_case__          =						English()
												snake_case__          =						_nlp.tokenizer
												snake_case__          =						ftfy.fix_text
									except ImportError:
												logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""       )
												snake_case__          =						BasicTokenizer(do_lower_case=__UpperCamelCase       )
												snake_case__          =						None

									with open(__UpperCamelCase					,   encoding="""utf-8"""       ) as vocab_handle:
												snake_case__          =						json.load(__UpperCamelCase       )
									snake_case__          =						{v: k for k, v in self.encoder.items()}
									with open(__UpperCamelCase					,   encoding="""utf-8"""       ) as merges_handle:
												snake_case__          =						merges_handle.read().split("""\n"""       )[1:-1]
									snake_case__          =						[tuple(merge.split()       ) for merge in merges]
									snake_case__          =						dict(zip(__UpperCamelCase					,   range(len(__UpperCamelCase       )       )       )       )
									snake_case__          =						{}






						@property
						def       __lowerCAmelCase(   self						:						str       ):



									'''simple docstring'''






									return True






						@property
						def       __lowerCAmelCase(   self						:						str       ):



									'''simple docstring'''






									return len(self.encoder       )






						def       __lowerCAmelCase(   self						:						Union[str, Any]       ):



									'''simple docstring'''






									return dict(self.encoder					,   **self.added_tokens_encoder       )






						def       __lowerCAmelCase(   self						:						Optional[Any]					,   __UpperCamelCase						:						Optional[int]       ):



									'''simple docstring'''






									snake_case__          =						tuple(token[:-1]       ) + (token[-1] + """</w>""",)
									if token in self.cache:
												return self.cache[token]
									snake_case__          =						get_pairs(__UpperCamelCase       )

									if not pairs:
												return token + "</w>"

									while True:
												snake_case__          =						min(__UpperCamelCase					,   key=lambda __UpperCamelCase       : self.bpe_ranks.get(__UpperCamelCase					,   float("""inf"""       )       )       )
												if bigram not in self.bpe_ranks:
															break
												snake_case__    ,			snake_case__          =						bigram
												snake_case__          =						[]
												snake_case__          =						0
												while i < len(__UpperCamelCase       ):
															try:
																		snake_case__          =						word.index(__UpperCamelCase					,   __UpperCamelCase       )
															except ValueError:
																		new_word.extend(word[i:]       )
																		break
															else:
																		new_word.extend(word[i:j]       )
																		snake_case__          =						j

															if word[i] == first and i < len(__UpperCamelCase       ) - 1 and word[i + 1] == second:
																		new_word.append(first + second       )
																		i += 2
															else:
																		new_word.append(word[i]       )
																		i += 1
												snake_case__          =						tuple(__UpperCamelCase       )
												snake_case__          =						new_word
												if len(__UpperCamelCase       ) == 1:
															break
												else:
															snake_case__          =						get_pairs(__UpperCamelCase       )
									snake_case__          =						""" """.join(__UpperCamelCase       )
									if word == "\n  </w>":
												snake_case__          =						"""\n</w>"""
									snake_case__          =						word
									return word






						def       __lowerCAmelCase(   self						:						str					,   __UpperCamelCase						:						Optional[int]       ):



									'''simple docstring'''






									snake_case__          =						[]
									if self.fix_text is None:
												# Using BERT's BasicTokenizer
												snake_case__          =						self.nlp.tokenize(__UpperCamelCase       )
												for token in text:
															split_tokens.extend(list(self.bpe(__UpperCamelCase       ).split(""" """       )       )       )
									else:
												# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
												snake_case__          =						self.nlp(text_standardize(self.fix_text(__UpperCamelCase       )       )       )
												for token in text:
															split_tokens.extend(list(self.bpe(token.text.lower()       ).split(""" """       )       )       )
									return split_tokens






						def       __lowerCAmelCase(   self						:						List[Any]					,   __UpperCamelCase						:						str       ):



									'''simple docstring'''






									return self.encoder.get(__UpperCamelCase					,   self.encoder.get(self.unk_token       )       )






						def       __lowerCAmelCase(   self						:						str					,   __UpperCamelCase						:						str       ):



									'''simple docstring'''






									return self.decoder.get(__UpperCamelCase					,   self.unk_token       )






						def       __lowerCAmelCase(   self						:						Union[str, Any]					,   __UpperCamelCase						:						int       ):



									'''simple docstring'''






									snake_case__          =						"""""".join(__UpperCamelCase       ).replace("""</w>"""					,   """ """       ).strip()
									return out_string







						def       __lowerCAmelCase(   self						:						Any					,   __UpperCamelCase						:						str					,   __UpperCamelCase						:						Optional[str] = None       ):



									'''simple docstring'''






									if not os.path.isdir(__UpperCamelCase       ):
												logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""       )
												return
									snake_case__          =						os.path.join(
									    __UpperCamelCase					,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )
									snake_case__          =						os.path.join(
									    __UpperCamelCase					,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]       )

									with open(__UpperCamelCase					,   """w"""					,   encoding="""utf-8"""       ) as f:
												f.write(json.dumps(self.encoder					,   indent=2					,   sort_keys=__UpperCamelCase					,   ensure_ascii=__UpperCamelCase       ) + """\n"""       )

									snake_case__          =						0
									with open(__UpperCamelCase					,   """w"""					,   encoding="""utf-8"""       ) as writer:
												writer.write("""#version: 0.2\n"""       )
												for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,   key=lambda __UpperCamelCase       : kv[1]       ):
															if index != token_index:
																		logger.warning(
																		    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																		    """ Please check that the tokenizer is not corrupted!"""       )
																		snake_case__          =						token_index
															writer.write(""" """.join(__UpperCamelCase       ) + """\n"""       )
															index += 1

									return vocab_file, merge_file