

'''simple docstring'''







import os
from shutil import copyfile
from typing import TYPE_CHECKING, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging
from ...utils.versions import require_version


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation

require_version('tokenizers>=0.13.3')

if is_sentencepiece_available():
							from .tokenization_llama import LlamaTokenizer
else:
							_lowerCamelCase    =  None

_lowerCamelCase    =  logging.get_logger(__name__)
_lowerCamelCase    =  {'vocab_file': 'tokenizer.model', 'tokenizer_file': 'tokenizer.json'}

_lowerCamelCase,							_lowerCamelCase    =  '[INST]', '[/INST]'
_lowerCamelCase,							_lowerCamelCase    =  '<<SYS>>\n', '\n<</SYS>>\n\n'

# fmt: off
_lowerCamelCase    =  'You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\\nanswers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\\nthat your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not\\ncorrect. If you don\'t know the answer to a question, please don\'t share false information.'
class 					__a   (		_snake_case  ):
						__SCREAMING_SNAKE_CASE     :					Tuple  =      VOCAB_FILES_NAMES
						__SCREAMING_SNAKE_CASE     :					int  =      LlamaTokenizer
						__SCREAMING_SNAKE_CASE     :					Tuple  =      'left'
						__SCREAMING_SNAKE_CASE     :					Any  =      ['input_ids', 'attention_mask']

						def __init__(		self	:						str    ,			lowercase__	:						Union[str, Any]=None    ,			lowercase__	:						Dict=None    ,			lowercase__	:						Union[str, Any]=False    ,			lowercase__	:						str="<unk>"    ,			lowercase__	:						int="<s>"    ,			lowercase__	:						int="</s>"    ,			lowercase__	:						int=True    ,			lowercase__	:						Optional[int]=False    ,			**lowercase__	:						int    ,			)						->Tuple:

							"""simple docstring"""



							super().__init__(
							    vocab_file=lowercase__    ,			tokenizer_file=lowercase__    ,			clean_up_tokenization_spaces=lowercase__    ,			unk_token=lowercase__    ,			bos_token=lowercase__    ,			eos_token=lowercase__    ,			**lowercase__    ,			)
							_lowercase							=     add_bos_token
							_lowercase							=     add_eos_token
							self.update_post_processor()

							_lowercase							=     vocab_file
							_lowercase							=     False if not self.vocab_file else True

						def 	_UpperCAmelCase							(		self	:						Optional[Any])						->str:

							"""simple docstring"""



							_lowercase							=     self.bos_token
							_lowercase							=     self.bos_token_id

							_lowercase							=     self.eos_token
							_lowercase							=     self.eos_token_id

							_lowercase							=     f"""{(bos+":0 ") * self.add_bos_token}$A:0{(" "+eos+":0") * self.add_eos_token}"""
							_lowercase							=     f"""{single}{(" "+bos+":1") * self.add_bos_token} $B:1{(" "+eos+":1") * self.add_eos_token}"""

							_lowercase							=     []
							if self.add_bos_token:
								special_tokens.append((bos, bos_token_id))
							if self.add_eos_token:
								special_tokens.append((eos, eos_token_id))
							_lowercase							=     processors.TemplateProcessing(
							    single=lowercase__    ,			pair=lowercase__    ,			special_tokens=lowercase__)

						@property
						def 	_UpperCAmelCase							(		self	:						Optional[Any])						->Optional[Any]:

							"""simple docstring"""



							return self._add_eos_token

						@property
						def 	_UpperCAmelCase							(		self	:						Tuple)						->Optional[Any]:

							"""simple docstring"""



							return self._add_bos_token

						@add_eos_token.setter
						def 	_UpperCAmelCase							(		self	:						Union[str, Any]    ,			lowercase__	:						str)						->int:

							"""simple docstring"""



							_lowercase							=     value
							self.update_post_processor()

						@add_bos_token.setter
						def 	_UpperCAmelCase							(		self	:						Tuple    ,			lowercase__	:						Tuple)						->Dict:

							"""simple docstring"""



							_lowercase							=     value
							self.update_post_processor()

						def 	_UpperCAmelCase							(		self	:						Dict    ,			lowercase__	:						str    ,			lowercase__	:						Optional[str] = None)						->Tuple[str]:

							"""simple docstring"""



							if not self.can_save_slow_tokenizer:
								raise ValueError(
								    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
								    """tokenizer.""")

							if not os.path.isdir(lowercase__):
								logger.error(f"""Vocabulary path ({save_directory}) should be a directory""")
								return
							_lowercase							=     os.path.join(
							    lowercase__    ,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""])

							if os.path.abspath(self.vocab_file) != os.path.abspath(lowercase__):
								copyfile(self.vocab_file    ,			lowercase__)

							return (out_vocab_file,)


						def 	_UpperCAmelCase							(		self	:						Dict    ,			lowercase__	:						"Conversation")						->Any:

							"""simple docstring"""



							_lowercase							=     list(conversation.iter_texts())
							if not all([is_user for is_user, msg in dialogue[::2]]) or not all(
							    [not is_user for is_user, msg in dialogue[1::2]]):
								raise ValueError(
								    """The model only supports 'user' and 'assistant' roles, starting with user and alternating (u/a/u/a/u...)""")

							_lowercase							=     []
							if len(conversation.past_user_inputs) > 0:
								if not conversation.past_user_inputs[0].startswith(lowercase__) or E_SYS not in conversation.past_user_inputs[0]:
									_lowercase							=     (
									    B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
									)
							elif not dialogue[0][1].startswith(lowercase__) or E_SYS not in dialogue[0][1]:
								_lowercase							=     (dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

							dialog_tokens += sum(
							    [
							        [self.bos_token_id]
							        + self.encode(
							            f"""{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} """    ,			add_special_tokens=lowercase__)
							        + [self.eos_token_id]
							        for prompt, answer in zip(dialogue[::2]    ,			dialogue[1::2])
							    ]    ,			[]    ,			)
							if not (dialogue[-1][0]):
								raise ValueError(f"""Last message must be from user, got {dialogue[-1]["role"]}""")
							dialog_tokens += [self.bos_token_id] + self.encode(
							    f"""{B_INST} {(dialogue[-1][1]).strip()} {E_INST}"""    ,			add_special_tokens=lowercase__)
							return dialog_tokens







'''simple docstring'''







class 					__a   :

						def __init__(		self	:						str    ,			lowercase__	:						List[str])						->List[str]:

							"""simple docstring"""



							_lowercase							=     val
							# Number of nodes in left subtree
							_lowercase							=     0
							_lowercase							=     None
							_lowercase							=     None
							_lowercase							=     None


						def 	_UpperCAmelCase							(		self	:						List[Any]    ,			lowercase__	:						Union[str, Any])						->str:

							"""simple docstring"""



							assert self.left_tree_size == other.left_tree_size, "Unequal Sizes of Blocks"

							if self.val < other.val:
								_lowercase							=     self.right
								_lowercase							=     None
								if self.right:
									_lowercase							=     other
								_lowercase							=     other
								_lowercase							=     self.left_tree_size * 2 + 1
								return self
							else:
								_lowercase							=     other.right
								_lowercase							=     None
								if other.right:
									_lowercase							=     self
								_lowercase							=     self
								_lowercase							=     other.left_tree_size * 2 + 1
								return other






class 					__a   :

						def __init__(		self	:						Optional[Any]    ,			lowercase__	:						List[Any]=None    ,			lowercase__	:						Any=None    ,			lowercase__	:						Any=0)						->Optional[int]:

							"""simple docstring"""



							_lowercase							=     heap_size
							_lowercase							=     bottom_root
							_lowercase							=     min_node

						def 	_UpperCAmelCase							(		self	:						Dict    ,			lowercase__	:						Union[str, Any])						->Any:

							"""simple docstring"""



							if other.size == 0:
								return None
							if self.size == 0:
								_lowercase							=     other.size
								_lowercase							=     other.bottom_root
								_lowercase							=     other.min_node
								return None
							# Update size
							_lowercase							=     self.size + other.size

							# Update min.node
							if self.min_node.val > other.min_node.val:
								_lowercase							=     other.min_node
							# Merge

							# Order roots by left_subtree_size
							_lowercase							=     []
							_lowercase		,		_lowercase							=     self.bottom_root, other.bottom_root
							while i or j:
								if i and ((not j) or i.left_tree_size < j.left_tree_size):
									combined_roots_list.append((i, True))
									_lowercase							=     i.parent
								else:
									combined_roots_list.append((j, False))
									_lowercase							=     j.parent
        # Insert links between them
							for i in range(len(lowercase__) - 1):
								if combined_roots_list[i][1] != combined_roots_list[i + 1][1]:
									_lowercase							=     combined_roots_list[i + 1][0]
									_lowercase							=     combined_roots_list[i][0]
        # Consecutively merge roots with same left_tree_size
							_lowercase							=     combined_roots_list[0][0]
							while i.parent:
								if (
								    (i.left_tree_size == i.parent.left_tree_size) and (not i.parent.parent)
								) or (
								    i.left_tree_size == i.parent.left_tree_size
								    and i.left_tree_size != i.parent.parent.left_tree_size
								):
									# Neighbouring Nodes
									_lowercase							=     i.left
									_lowercase							=     i.parent.parent

									# Merging trees
									_lowercase							=     i.merge_trees(i.parent)

									# Updating links
									_lowercase							=     previous_node
									_lowercase							=     next_node
									if previous_node:
										_lowercase							=     i
									if next_node:
										_lowercase							=     i
								else:
									_lowercase							=     i.parent
        # Updating self.bottom_root
							while i.left:
								_lowercase							=     i.left
							_lowercase							=     i

							# Update other
							_lowercase							=     self.size
							_lowercase							=     self.bottom_root
							_lowercase							=     self.min_node

							# Return the merged heap
							return self

						def 	_UpperCAmelCase							(		self	:						Optional[Any]    ,			lowercase__	:						List[Any])						->List[str]:

							"""simple docstring"""



							if self.size == 0:
								_lowercase							=     Node(lowercase__)
								_lowercase							=     1
								_lowercase							=     self.bottom_root
							else:
								# Create new node
								_lowercase							=     Node(lowercase__)

								# Update size
								self.size += 1

								# update min_node
								if val < self.min_node.val:
									_lowercase							=     new_node
								# Put new_node as a bottom_root in heap
								_lowercase							=     new_node
								_lowercase							=     self.bottom_root
								_lowercase							=     new_node

								# Consecutively merge roots with same left_tree_size
								while (
								    self.bottom_root.parent
								    and self.bottom_root.left_tree_size
								    == self.bottom_root.parent.left_tree_size
								):
									# Next node
									_lowercase							=     self.bottom_root.parent.parent

									# Merge
									_lowercase							=     self.bottom_root.merge_trees(self.bottom_root.parent)

									# Update Links
									_lowercase							=     next_node
									_lowercase							=     None
									if next_node:
										_lowercase							=     self.bottom_root

						def 	_UpperCAmelCase							(		self	:						Optional[Any])						->List[str]:

							"""simple docstring"""



							return self.min_node.val

						def 	_UpperCAmelCase							(		self	:						Optional[int])						->Tuple:

							"""simple docstring"""



							return self.size == 0

						def 	_UpperCAmelCase							(		self	:						str)						->List[str]:

							"""simple docstring"""



							_lowercase							=     self.min_node.val

							# Last element in heap corner case
							if self.size == 1:
								# Update size
								_lowercase							=     0

								# Update bottom root
								_lowercase							=     None

								# Update min_node
								_lowercase							=     None

								return min_value
							# No right subtree corner case
							# The structure of the tree implies that this should be the bottom root
							# and there is at least one other root
							if self.min_node.right is None:
								# Update size
								self.size -= 1

								# Update bottom root
								_lowercase							=     self.bottom_root.parent
								_lowercase							=     None

								# Update min_node
								_lowercase							=     self.bottom_root
								_lowercase							=     self.bottom_root.parent
								while i:
									if i.val < self.min_node.val:
										_lowercase							=     i
									_lowercase							=     i.parent
								return min_value
							# General case
							# Find the BinomialHeap of the right subtree of min_node
							_lowercase							=     self.min_node.right
							_lowercase							=     None
							_lowercase							=     bottom_of_new
							_lowercase							=     1

							# Size, min_node and bottom_root
							while bottom_of_new.left:
								_lowercase							=     size_of_new * 2 + 1
								_lowercase							=     bottom_of_new.left
								if bottom_of_new.val < min_of_new.val:
									_lowercase							=     bottom_of_new
        # Corner case of single root on top left path
							if (not self.min_node.left) and (not self.min_node.parent):
								_lowercase							=     size_of_new
								_lowercase							=     bottom_of_new
								_lowercase							=     min_of_new
								# print("Single root, multiple nodes case")
								return min_value
							# Remaining cases
							# Construct heap of right subtree
							_lowercase							=     BinomialHeap(
							    bottom_root=lowercase__    ,			min_node=lowercase__    ,			heap_size=lowercase__)

							# Update size
							_lowercase							=     self.size - 1 - size_of_new

							# Neighbour nodes
							_lowercase							=     self.min_node.left
							_lowercase							=     self.min_node.parent

							# Initialize new bottom_root and min_node
							_lowercase							=     previous_node or next_node
							_lowercase							=     next_node

							# Update links of previous_node and search below for new min_node and
							# bottom_root
							if previous_node:
								_lowercase							=     next_node

								# Update bottom_root and search for min_node below
								_lowercase							=     previous_node
								_lowercase							=     previous_node
								while self.bottom_root.left:
									_lowercase							=     self.bottom_root.left
									if self.bottom_root.val < self.min_node.val:
										_lowercase							=     self.bottom_root
							if next_node:
								_lowercase							=     previous_node

								# Search for new min_node above min_node
								_lowercase							=     next_node
								while i:
									if i.val < self.min_node.val:
										_lowercase							=     i
									_lowercase							=     i.parent
        # Merge heaps
							self.merge_heaps(lowercase__)

							return min_value

						def 	_UpperCAmelCase							(		self	:						List[str])						->str:

							"""simple docstring"""



							_lowercase							=     self.bottom_root
							while top_root.parent:
								_lowercase							=     top_root.parent
							# preorder
							_lowercase							=     []
							self.__traversal(lowercase__    ,			lowercase__)
							return heap_pre_order

						def 	_UpperCAmelCase							(		self	:						Dict    ,			lowercase__	:						Union[str, Any]    ,			lowercase__	:						List[Any]    ,			lowercase__	:						Optional[int]=0)						->int:

							"""simple docstring"""



							if curr_node:
								preorder.append((curr_node.val, level))
								self.__traversal(curr_node.left    ,			lowercase__    ,			level + 1)
								self.__traversal(curr_node.right    ,			lowercase__    ,			level + 1)
							else:
								preorder.append(("""#""", level))


						def __str__(		self	:						Optional[Any])						->str:

							"""simple docstring"""



							if self.is_empty():
								return ""
							_lowercase							=     self.pre_order()

							return "\n".join(("""-""" * level + str(lowercase__)) for value, level in preorder_heap)


# Unit Tests
if __name__ == "__main__":
							import doctest

							doctest.testmod()




