


"""simple docstring"""







import os
from shutil import copyfile
from typing import TYPE_CHECKING, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging
from ...utils.versions import require_version


if TYPE_CHECKING:
						from transformers.pipelines.conversational import Conversation

require_version("tokenizers>=0.13.3")

if is_sentencepiece_available():
						from .tokenization_llama import LlamaTokenizer
else:
						__a      			= None

__a      			= logging.get_logger(__name__)
__a      			= {"vocab_file": "tokenizer.model", "tokenizer_file": "tokenizer.json"}

__a			,						__a      			= "[INST]", "[/INST]"
__a			,						__a      			= "<<SYS>>\n", "\n<</SYS>>\n\n"

# fmt: off
__a      			= "You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\\nanswers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\\nthat your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not\\ncorrect. If you don't know the answer to a question, please don't share false information."
class 				lowerCamelCase							(      __SCREAMING_SNAKE_CASE			):


							"""simple docstring"""
							SCREAMING_SNAKE_CASE              =   VOCAB_FILES_NAMES
							SCREAMING_SNAKE_CASE              =   LlamaTokenizer
							SCREAMING_SNAKE_CASE              =   """left"""
							SCREAMING_SNAKE_CASE              =   ["""input_ids""", """attention_mask"""]






							def __init__(    self			:  Optional[Any]		,			lowerCamelCase_			:  Union[str, Any]=None		,			lowerCamelCase_			:  Dict=None		,			lowerCamelCase_			:  Dict=False		,			lowerCamelCase_			:  int="<unk>"		,			lowerCamelCase_			:  Dict="<s>"		,			lowerCamelCase_			:  int="</s>"		,			lowerCamelCase_			:  Optional[int]=True		,			lowerCamelCase_			:  List[Any]=False		,			**lowerCamelCase_			:  List[str]		,			):
													super().__init__(
													    vocab_file=lowerCamelCase_		,			tokenizer_file=lowerCamelCase_		,			clean_up_tokenization_spaces=lowerCamelCase_		,			unk_token=lowerCamelCase_		,			bos_token=lowerCamelCase_		,			eos_token=lowerCamelCase_		,			**lowerCamelCase_		,			)
													UpperCAmelCase__												=     add_bos_token
													UpperCAmelCase__												=     add_eos_token
													self.update_post_processor()

													UpperCAmelCase__												=     vocab_file
													UpperCAmelCase__												=     False if not self.vocab_file else True






							def        lowerCAmelCase  (    self			:  List[Any]					):
													UpperCAmelCase__												=     self.bos_token
													UpperCAmelCase__												=     self.bos_token_id

													UpperCAmelCase__												=     self.eos_token
													UpperCAmelCase__												=     self.eos_token_id

													UpperCAmelCase__												=     f"""{(bos+":0 ") * self.add_bos_token}$A:0{(" "+eos+":0") * self.add_eos_token}"""
													UpperCAmelCase__												=     f"""{single}{(" "+bos+":1") * self.add_bos_token} $B:1{(" "+eos+":1") * self.add_eos_token}"""

													UpperCAmelCase__												=     []
													if self.add_bos_token:
																			special_tokens.append((bos, bos_token_id)					)
													if self.add_eos_token:
																			special_tokens.append((eos, eos_token_id)					)
													UpperCAmelCase__												=     processors.TemplateProcessing(
													    single=lowerCamelCase_		,			pair=lowerCamelCase_		,			special_tokens=lowerCamelCase_					)






							@property
							def        lowerCAmelCase  (    self			:  Optional[int]					):
													return self._add_eos_token






							@property
							def        lowerCAmelCase  (    self			:  Tuple					):
													return self._add_bos_token






							@add_eos_token.setter
							def        lowerCAmelCase  (    self			:  str		,			lowerCamelCase_			:  Optional[Any]					):
													UpperCAmelCase__												=     value
													self.update_post_processor()






							@add_bos_token.setter
							def        lowerCAmelCase  (    self			:  Optional[Any]		,			lowerCamelCase_			:  Any					):
													UpperCAmelCase__												=     value
													self.update_post_processor()






							def        lowerCAmelCase  (    self			:  List[Any]		,			lowerCamelCase_			:  str		,			lowerCamelCase_			:  Optional[str] = None					):
													if not self.can_save_slow_tokenizer:
																			raise ValueError(
																			    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
																			    '''tokenizer.'''					)

													if not os.path.isdir(lowerCamelCase_					):
																			logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""					)
																			return
													UpperCAmelCase__												=     os.path.join(
													    lowerCamelCase_		,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

													if os.path.abspath(self.vocab_file					) != os.path.abspath(lowerCamelCase_					):
																			copyfile(self.vocab_file		,			lowerCamelCase_					)

													return (out_vocab_file,)






							def        lowerCAmelCase  (    self			:  Optional[int]		,			lowerCamelCase_			:  "Conversation"					):
													UpperCAmelCase__												=     list(conversation.iter_texts()					)
													if not all([is_user for is_user, msg in dialogue[::2]]					) or not all(
													    [not is_user for is_user, msg in dialogue[1::2]]					):
																			raise ValueError(
																			    '''The model only supports \'user\' and \'assistant\' roles, starting with user and alternating (u/a/u/a/u...)'''					)

													UpperCAmelCase__												=     []
													if len(conversation.past_user_inputs					) > 0:
																			if not conversation.past_user_inputs[0].startswith(lowerCamelCase_					) or E_SYS not in conversation.past_user_inputs[0]:
																									UpperCAmelCase__												=     (
																									    B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
																									)
													elif not dialogue[0][1].startswith(lowerCamelCase_					) or E_SYS not in dialogue[0][1]:
																			UpperCAmelCase__												=     (dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

													dialog_tokens += sum(
													    [
													        [self.bos_token_id]
													        + self.encode(
													            f"""{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} """		,			add_special_tokens=lowerCamelCase_					)
													        + [self.eos_token_id]
													        for prompt, answer in zip(dialogue[::2]		,			dialogue[1::2]					)
													    ]		,			[]		,			)
													if not (dialogue[-1][0]):
																			raise ValueError(f"""Last message must be from user, got {dialogue[-1]["role"]}"""					)
													dialog_tokens += [self.bos_token_id] + self.encode(
													    f"""{B_INST} {(dialogue[-1][1]).strip()} {E_INST}"""		,			add_special_tokens=lowerCamelCase_					)
													return dialog_tokens



"""simple docstring"""







import inspect
import json
import os
import tempfile
import unittest

from transformers.models.speech_to_text_a import SpeechaTextaTokenizer
from transformers.models.speech_to_text_a.tokenization_speech_to_text_a import VOCAB_FILES_NAMES

from ...test_tokenization_common import TokenizerTesterMixin
class 				lowerCamelCase							(      __SCREAMING_SNAKE_CASE					,       unittest.TestCase			):


							"""simple docstring"""
							SCREAMING_SNAKE_CASE              =   SpeechaTextaTokenizer
							SCREAMING_SNAKE_CASE              =   False






							def        lowerCAmelCase  (    self			:  Dict					):
													super().setUp()

													UpperCAmelCase__												=     '''<s> <pad> </s> <unk> here@@ a couple of@@ words for the he@@ re@@ vocab'''.split(''' '''					)
													UpperCAmelCase__												=     ['''he re</w> 123''', '''here a 1456''']
													UpperCAmelCase__												=     dict(zip(lowerCamelCase_		,			range(len(lowerCamelCase_					)					)					)					)

													UpperCAmelCase__												=     {'''pad_token''': '''<pad>''', '''unk_token''': '''<unk>''', '''bos_token''': '''<s>''', '''eos_token''': '''</s>'''}

													UpperCAmelCase__												=     tempfile.mkdtemp()
													UpperCAmelCase__												=     os.path.join(self.tmpdirname		,			VOCAB_FILES_NAMES['''vocab_file''']					)
													UpperCAmelCase__												=     os.path.join(self.tmpdirname		,			VOCAB_FILES_NAMES['''merges_file''']					)

													with open(self.vocab_file		,			'''w'''		,			encoding='''utf-8'''					) as fp:
																			fp.write(json.dumps(lowerCamelCase_					) + '''\n'''					)

													with open(self.merges_file		,			'''w'''					) as fp:
																			fp.write('''\n'''.join(lowerCamelCase_					)					)






							def        lowerCAmelCase  (    self			:  Union[str, Any]					):
													UpperCAmelCase__												=     list(self.get_tokenizer().get_vocab().keys()					)

													self.assertEqual(vocab_keys[0]		,			'''<s>'''					)
													self.assertEqual(vocab_keys[1]		,			'''<pad>'''					)
													self.assertEqual(vocab_keys[-1]		,			'''vocab'''					)
													self.assertEqual(len(lowerCamelCase_					)		,			14					)






							def        lowerCAmelCase  (    self			:  Optional[Any]					):
													self.assertEqual(self.get_tokenizer().vocab_size		,			14					)






							def        lowerCAmelCase  (    self			:  Any					):
													UpperCAmelCase__												=     SpeechaTextaTokenizer.from_pretrained(self.tmpdirname					)

													# make sure @@ is correctly concatenated
													UpperCAmelCase__												=     [4, 6, 8, 7, 10]  # ["here@@", "couple", "words", "of@@", "the"]
													UpperCAmelCase__												=     tokenizer.decode(lowerCamelCase_					)

													self.assertTrue(output_string == '''herecouple words ofthe'''					)






							def        lowerCAmelCase  (    self			:  int					):
													UpperCAmelCase__												=     SpeechaTextaTokenizer.from_pretrained(self.tmpdirname					)

													with tempfile.TemporaryDirectory() as tmp_dirname:
																			tokenizer.save_pretrained(lowerCamelCase_					)
																			os.remove(os.path.join(lowerCamelCase_		,			'''merges.txt'''					)					)

																			# load tokenizer without merges file should not throw an error
																			UpperCAmelCase__												=     SpeechaTextaTokenizer.from_pretrained(lowerCamelCase_					)

													with tempfile.TemporaryDirectory() as tmp_dirname:
																			# save tokenizer and load again
																			tokenizer.save_pretrained(lowerCamelCase_					)
																			UpperCAmelCase__												=     SpeechaTextaTokenizer.from_pretrained(lowerCamelCase_					)

													self.assertIsNotNone(lowerCamelCase_					)






							def        lowerCAmelCase  (    self			:  int					):
													if not self.test_slow_tokenizer:
																			return

													UpperCAmelCase__												=     inspect.signature(self.tokenizer_class.__init__					)
													UpperCAmelCase__												=     self.get_tokenizer()

													for parameter_name, parameter in signature.parameters.items():
																			if parameter.default != inspect.Parameter.empty and parameter_name != "merges_file":
																									self.assertIn(lowerCamelCase_		,			tokenizer.init_kwargs					)