






import unittest

from transformers import MarkupLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    MarkupLMForQuestionAnswering,
				    MarkupLMForSequenceClassification,
				    MarkupLMForTokenClassification,
				    MarkupLMModel,
				)

# TODO check dependencies
from transformers import MarkupLMFeatureExtractor, MarkupLMProcessor, MarkupLMTokenizer


class 				__a				:





		'''simple docstring'''



		def __init__(    self			,     UpperCamelCase__			,     UpperCamelCase__=13			,     UpperCamelCase__=7			,     UpperCamelCase__=True			,     UpperCamelCase__=True			,     UpperCamelCase__=True			,     UpperCamelCase__=True			,     UpperCamelCase__=99			,     UpperCamelCase__=32			,     UpperCamelCase__=5			,     UpperCamelCase__=4			,     UpperCamelCase__=37			,     UpperCamelCase__="gelu"			,     UpperCamelCase__=0.1			,     UpperCamelCase__=0.1			,     UpperCamelCase__=512			,     UpperCamelCase__=16			,     UpperCamelCase__=2			,     UpperCamelCase__=0.02			,     UpperCamelCase__=3			,     UpperCamelCase__=None			,     UpperCamelCase__=20			,     UpperCamelCase__=30			,     UpperCamelCase__=2			,     UpperCamelCase__=2			,     UpperCamelCase__=10			,     ):
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= parent
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= batch_size
						SCREAMING_SNAKE_CASE_ :						int    				= seq_length
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= is_training
						SCREAMING_SNAKE_CASE_ :						Dict    				= use_input_mask
						SCREAMING_SNAKE_CASE_ :						Dict    				= use_token_type_ids
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= use_labels
						SCREAMING_SNAKE_CASE_ :						int    				= vocab_size
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= hidden_size
						SCREAMING_SNAKE_CASE_ :						Any    				= num_hidden_layers
						SCREAMING_SNAKE_CASE_ :						int    				= num_attention_heads
						SCREAMING_SNAKE_CASE_ :						Any    				= intermediate_size
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= hidden_act
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= hidden_dropout_prob
						SCREAMING_SNAKE_CASE_ :						List[str]    				= attention_probs_dropout_prob
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= max_position_embeddings
						SCREAMING_SNAKE_CASE_ :						Tuple    				= type_vocab_size
						SCREAMING_SNAKE_CASE_ :						str    				= type_sequence_label_size
						SCREAMING_SNAKE_CASE_ :						Dict    				= initializer_range
						SCREAMING_SNAKE_CASE_ :						int    				= num_labels
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= scope
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= max_xpath_tag_unit_embeddings
						SCREAMING_SNAKE_CASE_ :						Tuple    				= max_xpath_subs_unit_embeddings
						SCREAMING_SNAKE_CASE_ :						Dict    				= tag_pad_id
						SCREAMING_SNAKE_CASE_ :						str    				= subs_pad_id
						SCREAMING_SNAKE_CASE_ :						Tuple    				= max_depth


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= ids_tensor([self.batch_size, self.seq_length]			,     self.vocab_size							)

						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= ids_tensor(
						    [self.batch_size, self.seq_length, self.max_depth]			,     self.max_xpath_tag_unit_embeddings							)

						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= ids_tensor(
						    [self.batch_size, self.seq_length, self.max_depth]			,     self.max_xpath_subs_unit_embeddings							)

						SCREAMING_SNAKE_CASE_ :						Any    				= None
						if self.use_input_mask:
										SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= ids_tensor([self.batch_size, self.seq_length]			,     vocab_size=2							)

						SCREAMING_SNAKE_CASE_ :						Dict    				= None
						if self.use_token_type_ids:
										SCREAMING_SNAKE_CASE_ :						int    				= ids_tensor([self.batch_size, self.seq_length]			,     self.type_vocab_size							)

						SCREAMING_SNAKE_CASE_ :						int    				= None
						SCREAMING_SNAKE_CASE_ :						Tuple    				= None
						if self.use_labels:
										SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= ids_tensor([self.batch_size]			,     self.type_sequence_label_size							)
										SCREAMING_SNAKE_CASE_ :						Any    				= ids_tensor([self.batch_size, self.seq_length]			,     self.num_labels							)

						SCREAMING_SNAKE_CASE_ :						Tuple    				= self.get_config()

						return (
						    config,
						    input_ids,
						    xpath_tags_seq,
						    xpath_subs_seq,
						    token_type_ids,
						    input_mask,
						    sequence_labels,
						    token_labels,
						)


		def 	__snake_case      (    self							):
						return MarkupLMConfig(
						    vocab_size=self.vocab_size			,     hidden_size=self.hidden_size			,     num_hidden_layers=self.num_hidden_layers			,     num_attention_heads=self.num_attention_heads			,     intermediate_size=self.intermediate_size			,     hidden_act=self.hidden_act			,     hidden_dropout_prob=self.hidden_dropout_prob			,     attention_probs_dropout_prob=self.attention_probs_dropout_prob			,     max_position_embeddings=self.max_position_embeddings			,     type_vocab_size=self.type_vocab_size			,     initializer_range=self.initializer_range			,     max_xpath_tag_unit_embeddings=self.max_xpath_tag_unit_embeddings			,     max_xpath_subs_unit_embeddings=self.max_xpath_subs_unit_embeddings			,     tag_pad_id=self.tag_pad_id			,     subs_pad_id=self.subs_pad_id			,     max_depth=self.max_depth			,     )


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     ):
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= MarkupLMModel(config=UpperCamelCase__							)
						model.to(UpperCamelCase__							)
						model.eval()
						print('Configs:'			,     model.config.tag_pad_id			,     model.config.subs_pad_id							)
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= model(UpperCamelCase__			,     attention_mask=UpperCamelCase__			,     token_type_ids=UpperCamelCase__							)
						SCREAMING_SNAKE_CASE_ :						List[str]    				= model(UpperCamelCase__			,     token_type_ids=UpperCamelCase__							)
						SCREAMING_SNAKE_CASE_ :						int    				= model(UpperCamelCase__							)
						self.parent.assertEqual(result.last_hidden_state.shape			,     (self.batch_size, self.seq_length, self.hidden_size)							)
						self.parent.assertEqual(result.pooler_output.shape			,     (self.batch_size, self.hidden_size)							)


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     ):
						SCREAMING_SNAKE_CASE_ :						Dict    				= self.num_labels
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= MarkupLMForSequenceClassification(UpperCamelCase__							)
						model.to(UpperCamelCase__							)
						model.eval()
						SCREAMING_SNAKE_CASE_ :						str    				= model(
						    UpperCamelCase__			,     xpath_tags_seq=UpperCamelCase__			,     xpath_subs_seq=UpperCamelCase__			,     attention_mask=UpperCamelCase__			,     token_type_ids=UpperCamelCase__			,     labels=UpperCamelCase__			,     )
						self.parent.assertEqual(result.logits.shape			,     (self.batch_size, self.num_labels)							)


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     ):
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= self.num_labels
						SCREAMING_SNAKE_CASE_ :						List[str]    				= MarkupLMForTokenClassification(config=UpperCamelCase__							)
						model.to(UpperCamelCase__							)
						model.eval()
						SCREAMING_SNAKE_CASE_ :						str    				= model(
						    UpperCamelCase__			,     xpath_tags_seq=UpperCamelCase__			,     xpath_subs_seq=UpperCamelCase__			,     attention_mask=UpperCamelCase__			,     token_type_ids=UpperCamelCase__			,     labels=UpperCamelCase__			,     )
						self.parent.assertEqual(result.logits.shape			,     (self.batch_size, self.seq_length, self.num_labels)							)


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     ):
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= MarkupLMForQuestionAnswering(config=UpperCamelCase__							)
						model.to(UpperCamelCase__							)
						model.eval()
						SCREAMING_SNAKE_CASE_ :						int    				= model(
						    UpperCamelCase__			,     xpath_tags_seq=UpperCamelCase__			,     xpath_subs_seq=UpperCamelCase__			,     attention_mask=UpperCamelCase__			,     token_type_ids=UpperCamelCase__			,     start_positions=UpperCamelCase__			,     end_positions=UpperCamelCase__			,     )
						self.parent.assertEqual(result.start_logits.shape			,     (self.batch_size, self.seq_length)							)
						self.parent.assertEqual(result.end_logits.shape			,     (self.batch_size, self.seq_length)							)






		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Any    				= self.prepare_config_and_inputs()
						(
						    (
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							
						) :						int    				= config_and_inputs
						SCREAMING_SNAKE_CASE_ :						Any    				= {
						    'input_ids': input_ids,
						    'xpath_tags_seq': xpath_tags_seq,
						    'xpath_subs_seq': xpath_subs_seq,
						    'token_type_ids': token_type_ids,
						    'attention_mask': input_mask,
						}
						return config, inputs_dict






@require_torch
class 				__a				(	__A			,						__A			,						unittest.TestCase    ):





		'''simple docstring'''



		UpperCAmelCase__   :					Dict					     =    (
		    (
		        MarkupLMModel,
		        MarkupLMForSequenceClassification,
		        MarkupLMForTokenClassification,
		        MarkupLMForQuestionAnswering,
		    )
		    if is_torch_available()
		    else None
		)
		UpperCAmelCase__   :					Union[str, Any]					     =    (
		    {
		        """feature-extraction""": MarkupLMModel,
		        """question-answering""": MarkupLMForQuestionAnswering,
		        """text-classification""": MarkupLMForSequenceClassification,
		        """token-classification""": MarkupLMForTokenClassification,
		        """zero-shot""": MarkupLMForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__							):
						# ValueError: Nodes must be of type `List[str]` (single pretokenized example), or `List[List[str]]`
						# (batch of pretokenized examples).
						return True


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						int    				= MarkupLMModelTester(self							)
						SCREAMING_SNAKE_CASE_ :						Any    				= ConfigTester(self			,     config_class=UpperCamelCase__			,     hidden_size=37							)


		def 	__snake_case      (    self							):
						self.config_tester.run_common_tests()


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*UpperCamelCase__							)


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_sequence_classification(*UpperCamelCase__							)


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Any    				= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_token_classification(*UpperCamelCase__							)






		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_question_answering(*UpperCamelCase__							)






def        _lowerCamelCase(      ):





				'''simple docstring'''







				SCREAMING_SNAKE_CASE_ :						int    				= '\n    <!DOCTYPE html>\n    <html>\n    <head>\n    <title>Page Title</title>\n    </head>\n    <body>\n\n    <h1>This is a Heading</h1>\n    <p>This is a paragraph.</p>\n\n    </body>\n    </html>\n    '

				return html_string


@require_torch
class 				__a				(	unittest.TestCase    ):





		'''simple docstring'''



		@cached_property
		def 	__snake_case      (    self							):
						# TODO use from_pretrained here
						SCREAMING_SNAKE_CASE_ :						List[str]    				= MarkupLMFeatureExtractor()
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= MarkupLMTokenizer.from_pretrained('microsoft/markuplm-base'							)

						return MarkupLMProcessor(UpperCamelCase__			,     UpperCamelCase__							)






		@slow
		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Tuple    				= MarkupLMModel.from_pretrained('microsoft/markuplm-base'							).to(UpperCamelCase__							)

						SCREAMING_SNAKE_CASE_ :						Tuple    				= self.default_processor

						SCREAMING_SNAKE_CASE_ :						int    				= processor(prepare_html_string()			,     return_tensors='pt'							)
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= inputs.to(UpperCamelCase__							)

						# forward pass
						with torch.no_grad():
										SCREAMING_SNAKE_CASE_ :						str    				= model(**UpperCamelCase__							)

						# verify the last hidden states
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= torch.Size([1, 14, 768]							)
						self.assertEqual(outputs.last_hidden_state.shape			,     UpperCamelCase__							)

						SCREAMING_SNAKE_CASE_ :						Tuple    				= torch.tensor(
						    [[0.06_75, -0.00_52, 0.50_01], [-0.22_81, 0.08_02, 0.21_92], [-0.05_83, -0.33_11, 0.11_85]]							).to(UpperCamelCase__							)

						self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]			,     UpperCamelCase__			,     atol=1E-4							)							)







import os
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
				from transformers.pipelines.conversational import Conversation

A										=      logging.get_logger(__name__)

A										=      {'vocab_file': 'tokenizer.model'}

A										=      {
    'vocab_file': {
        'hf-internal-testing/llama-tokenizer': 'https://huggingface.co/hf-internal-testing/llama-tokenizer/resolve/main/tokenizer.model',
    },
    'tokenizer_file': {
        'hf-internal-testing/llama-tokenizer': 'https://huggingface.co/hf-internal-testing/llama-tokenizer/resolve/main/tokenizer_config.json',
    },
}
A										=      {
    'hf-internal-testing/llama-tokenizer': 2048,
}
A										=      '▁'

A			,	A										=      '[INST]', '[/INST]'
A			,	A										=      '<<SYS>>\n', '\n<</SYS>>\n\n'

# fmt: off
A										=      'You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\\nanswers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\\nthat your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not\\ncorrect. If you don\'t know the answer to a question, please don\'t share false information.'


class 				__a				(	__A    ):





		'''simple docstring'''



		UpperCAmelCase__   :					Optional[int]					     =    VOCAB_FILES_NAMES
		UpperCAmelCase__   :					Dict					     =    PRETRAINED_VOCAB_FILES_MAP
		UpperCAmelCase__   :					int					     =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		UpperCAmelCase__   :					Union[str, Any]					     =    ["""input_ids""", """attention_mask"""]


		def __init__(    self			,     UpperCamelCase__			,     UpperCamelCase__="<unk>"			,     UpperCamelCase__="<s>"			,     UpperCamelCase__="</s>"			,     UpperCamelCase__=None			,     UpperCamelCase__ = None			,     UpperCamelCase__=True			,     UpperCamelCase__=False			,     UpperCamelCase__=False			,     UpperCamelCase__=True			,     **UpperCamelCase__			,     ):
						SCREAMING_SNAKE_CASE_ :						str    				= {} if sp_model_kwargs is None else sp_model_kwargs
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= AddedToken(UpperCamelCase__			,     lstrip=UpperCamelCase__			,     rstrip=UpperCamelCase__							) if isinstance(UpperCamelCase__			,     UpperCamelCase__							) else bos_token
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= AddedToken(UpperCamelCase__			,     lstrip=UpperCamelCase__			,     rstrip=UpperCamelCase__							) if isinstance(UpperCamelCase__			,     UpperCamelCase__							) else eos_token
						SCREAMING_SNAKE_CASE_ :						str    				= AddedToken(UpperCamelCase__			,     lstrip=UpperCamelCase__			,     rstrip=UpperCamelCase__							) if isinstance(UpperCamelCase__			,     UpperCamelCase__							) else unk_token
						SCREAMING_SNAKE_CASE_ :						int    				= AddedToken(UpperCamelCase__			,     lstrip=UpperCamelCase__			,     rstrip=UpperCamelCase__							) if isinstance(UpperCamelCase__			,     UpperCamelCase__							) else pad_token
						super().__init__(
						    bos_token=UpperCamelCase__			,     eos_token=UpperCamelCase__			,     unk_token=UpperCamelCase__			,     pad_token=UpperCamelCase__			,     add_bos_token=UpperCamelCase__			,     add_eos_token=UpperCamelCase__			,     sp_model_kwargs=self.sp_model_kwargs			,     clean_up_tokenization_spaces=UpperCamelCase__			,     legacy=UpperCamelCase__			,     **UpperCamelCase__			,     )
						if legacy:
										logger.warning_once(
										    F'''You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to'''
										    ' read the related pull request available at https://github.com/huggingface/transformers/pull/24565'							)
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= legacy
						SCREAMING_SNAKE_CASE_ :						Dict    				= vocab_file
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= add_bos_token
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= add_eos_token
						SCREAMING_SNAKE_CASE_ :						Any    				= spm.SentencePieceProcessor(**self.sp_model_kwargs							)
						self.sp_model.Load(UpperCamelCase__							)


		def __getstate__(    self							):
						SCREAMING_SNAKE_CASE_ :						List[str]    				= self.__dict__.copy()
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= None
						SCREAMING_SNAKE_CASE_ :						Any    				= self.sp_model.serialized_model_proto()
						return state


		def __setstate__(    self			,     UpperCamelCase__							):
						SCREAMING_SNAKE_CASE_ :						str    				= d
						SCREAMING_SNAKE_CASE_ :						str    				= spm.SentencePieceProcessor(**self.sp_model_kwargs							)
						self.sp_model.LoadFromSerializedProto(self.sp_model_proto							)


		@property
		def 	__snake_case      (    self							):
						return self.sp_model.get_piece_size()


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Tuple    				= {self.convert_ids_to_tokens(UpperCamelCase__							): i for i in range(self.vocab_size							)}
						vocab.update(self.added_tokens_encoder							)
						return vocab


		def 	__snake_case      (    self			,     UpperCamelCase__			,     **UpperCamelCase__							):
						# Replace the SPIECE_UNDERLINE with a space to make sure SPIECE_UNDERLINE is only used at
						# the beginning of the text
						if not self.legacy:
										SCREAMING_SNAKE_CASE_ :						List[str]    				= SPIECE_UNDERLINE + text.replace(UpperCamelCase__			,     ' '							)
						return super().tokenize(UpperCamelCase__			,     **UpperCamelCase__							)


		def 	__snake_case      (    self			,     UpperCamelCase__							):
						if not self.legacy:
										SCREAMING_SNAKE_CASE_ :						Any    				= text.startswith(UpperCamelCase__							)
										if is_first:
														SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= text[1:]

						SCREAMING_SNAKE_CASE_ :						Tuple    				= self.sp_model.encode(UpperCamelCase__			,     out_type=UpperCamelCase__							)

						if not self.legacy and not is_first and not text.startswith(' '							) and tokens[0].startswith(UpperCamelCase__							):
										SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= ([tokens[0][1:]] if len(tokens[0]							) > 1 else []) + tokens[1:]
						return tokens


		def 	__snake_case      (    self			,     UpperCamelCase__							):
						return self.sp_model.piece_to_id(UpperCamelCase__							)


		def 	__snake_case      (    self			,     UpperCamelCase__							):
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= self.sp_model.IdToPiece(UpperCamelCase__							)
						return token


		def 	__snake_case      (    self			,     UpperCamelCase__							):
						SCREAMING_SNAKE_CASE_ :						List[str]    				= []
						SCREAMING_SNAKE_CASE_ :						int    				= ''
						SCREAMING_SNAKE_CASE_ :						List[str]    				= False
						for i, token in enumerate(UpperCamelCase__							):
										# make sure that special tokens are not decoded using sentencepiece model
										if token in self.all_special_tokens:
														if not prev_is_special and i != 0:
																		out_string += " "
														out_string += self.sp_model.decode(UpperCamelCase__							) + token
														SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= True
														SCREAMING_SNAKE_CASE_ :						str    				= []
										else:
														current_sub_tokens.append(UpperCamelCase__							)
														SCREAMING_SNAKE_CASE_ :						int    				= False
						out_string += self.sp_model.decode(UpperCamelCase__							)
						return out_string


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__ = None							):
						if not os.path.isdir(UpperCamelCase__							):
										logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''							)
										return
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= os.path.join(
						    UpperCamelCase__			,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']							)

						if os.path.abspath(self.vocab_file							) != os.path.abspath(UpperCamelCase__							) and os.path.isfile(self.vocab_file							):
										copyfile(self.vocab_file			,     UpperCamelCase__							)
						elif not os.path.isfile(self.vocab_file							):
										with open(UpperCamelCase__			,     'wb'							) as fi:
														SCREAMING_SNAKE_CASE_ :						Dict    				= self.sp_model.serialized_model_proto()
														fi.write(UpperCamelCase__							)

						return (out_vocab_file,)


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__=None							):
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= [self.bos_token_id] if self.add_bos_token else []
						SCREAMING_SNAKE_CASE_ :						List[str]    				= [self.eos_token_id] if self.add_eos_token else []

						SCREAMING_SNAKE_CASE_ :						int    				= bos_token_id + token_ids_a + eos_token_id

						if token_ids_a is not None:
										SCREAMING_SNAKE_CASE_ :						int    				= output + bos_token_id + token_ids_a + eos_token_id

						return output


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__ = None			,     UpperCamelCase__ = False							):
						if already_has_special_tokens:
										return super().get_special_tokens_mask(
										    token_ids_a=UpperCamelCase__			,     token_ids_a=UpperCamelCase__			,     already_has_special_tokens=UpperCamelCase__							)

						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= [1] if self.add_bos_token else []
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= [1] if self.add_eos_token else []

						if token_ids_a is None:
										return bos_token_id + ([0] * len(UpperCamelCase__							)) + eos_token_id
						return (
						    bos_token_id
						    + ([0] * len(UpperCamelCase__							))
						    + eos_token_id
						    + bos_token_id
						    + ([0] * len(UpperCamelCase__							))
						    + eos_token_id
						)


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__ = None							):
						SCREAMING_SNAKE_CASE_ :						Any    				= [self.bos_token_id] if self.add_bos_token else []
						SCREAMING_SNAKE_CASE_ :						Tuple    				= [self.eos_token_id] if self.add_eos_token else []

						SCREAMING_SNAKE_CASE_ :						Any    				= [0] * len(bos_token_id + token_ids_a + eos_token_id							)

						if token_ids_a is not None:
										output += [1] * len(bos_token_id + token_ids_a + eos_token_id							)

						return output






		def 	__snake_case      (    self			,     UpperCamelCase__							):
						SCREAMING_SNAKE_CASE_ :						int    				= list(conversation.iter_texts()							)
						if not all([is_user for is_user, msg in dialogue[::2]]							) or not all(
						    [not is_user for is_user, msg in dialogue[1::2]]							):
										raise ValueError(
										    'The model only supports \'user\' and \'assistant\' roles, starting with user and alternating (u/a/u/a/u...)'							)

						SCREAMING_SNAKE_CASE_ :						List[int]    				= []
						if len(conversation.past_user_inputs							) > 0:
										if not conversation.past_user_inputs[0].startswith(UpperCamelCase__							) or E_SYS not in conversation.past_user_inputs[0]:
														SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= (
														    B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
														)
						elif not dialogue[0][1].startswith(UpperCamelCase__							) or E_SYS not in dialogue[0][1]:
										SCREAMING_SNAKE_CASE_ :						Tuple    				= (dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

						dialog_tokens += sum(
						    [
						        [self.bos_token_id]
						        + self.encode(
						            F'''{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} '''			,     add_special_tokens=UpperCamelCase__							)
						        + [self.eos_token_id]
						        for prompt, answer in zip(dialogue[::2]			,     dialogue[1::2]							)
						    ]			,     []			,     )
						if not (dialogue[-1][0]):
										raise ValueError(F'''Last message must be from user, got {dialogue[-1]['role']}'''							)
						dialog_tokens += [self.bos_token_id] + self.encode(
						    F'''{B_INST} {(dialogue[-1][1]).strip()} {E_INST}'''			,     add_special_tokens=UpperCamelCase__							)
						return dialog_tokens