
"""simple docstring"""






import os
from shutil import copyfile
from typing import TYPE_CHECKING, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging
from ...utils.versions import require_version


if TYPE_CHECKING:
      from transformers.pipelines.conversational import Conversation

require_version('tokenizers>=0.13.3')

if is_sentencepiece_available():
      from .tokenization_llama import LlamaTokenizer
else:
      UpperCamelCase_						   =   None

UpperCamelCase_						   =   logging.get_logger(__name__)
UpperCamelCase_						   =   {'vocab_file': 'tokenizer.model', 'tokenizer_file': 'tokenizer.json'}

UpperCamelCase_   ,     UpperCamelCase_						   =   '[INST]', '[/INST]'
UpperCamelCase_   ,     UpperCamelCase_						   =   '<<SYS>>\n', '\n<</SYS>>\n\n'

# fmt: off
UpperCamelCase_						   =   'You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\\nanswers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\\nthat your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not\\ncorrect. If you don\'t know the answer to a question, please don\'t share false information.'





class 							snake_case       (     SCREAMING_SNAKE_CASE_		):
   a_					:		List[str]     					=     VOCAB_FILES_NAMES
   a_					:		Dict     					=     LlamaTokenizer
   a_					:		List[str]     					=     """left"""
   a_					:		Any     					=     ["""input_ids""", """attention_mask"""]







   def __init__( self       ,							__UpperCAmelCase=None       ,							__UpperCAmelCase=None       ,							__UpperCAmelCase=False       ,							__UpperCAmelCase="<unk>"       ,							__UpperCAmelCase="<s>"       ,							__UpperCAmelCase="</s>"       ,							__UpperCAmelCase=True       ,							__UpperCAmelCase=False       ,							**__UpperCAmelCase       ,							)   ->Any:
      super().__init__(
          vocab_file=__UpperCAmelCase       ,							tokenizer_file=__UpperCAmelCase       ,							clean_up_tokenization_spaces=__UpperCAmelCase       ,							unk_token=__UpperCAmelCase       ,							bos_token=__UpperCAmelCase       ,							eos_token=__UpperCAmelCase       ,							**__UpperCAmelCase       ,							)
      a_						   =     add_bos_token
      a_						   =     add_eos_token
      self.update_post_processor()

      a_						   =     vocab_file
      a_						   =     False if not self.vocab_file else True







   def    UpperCAmelCase__	( self)   ->Union[str, Any]:
      a_						   =     self.bos_token
      a_						   =     self.bos_token_id

      a_						   =     self.eos_token
      a_						   =     self.eos_token_id

      a_						   =     F'''{(bos+":0 ") * self.add_bos_token}$A:0{(" "+eos+":0") * self.add_eos_token}'''
      a_						   =     F'''{single}{(" "+bos+":1") * self.add_bos_token} $B:1{(" "+eos+":1") * self.add_eos_token}'''

      a_						   =     []
      if self.add_bos_token:
         special_tokens.append((bos, bos_token_id))
      if self.add_eos_token:
         special_tokens.append((eos, eos_token_id))
      a_						   =     processors.TemplateProcessing(
          single=__UpperCAmelCase       ,							pair=__UpperCAmelCase       ,							special_tokens=__UpperCAmelCase)







   @property
   def    UpperCAmelCase__	( self)   ->List[str]:
      return self._add_eos_token







   @property
   def    UpperCAmelCase__	( self)   ->Any:
      return self._add_bos_token







   @add_eos_token.setter
   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Union[str, Any]:
      a_						   =     value
      self.update_post_processor()







   @add_bos_token.setter
   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Union[str, Any]:
      a_						   =     value
      self.update_post_processor()







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase       ,							__UpperCAmelCase = None)   ->Tuple[str]:
      if not self.can_save_slow_tokenizer:
         raise ValueError(
             "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
             "tokenizer.")

      if not os.path.isdir(__UpperCAmelCase):
         logger.error(F'''Vocabulary path ({save_directory}) should be a directory''')
         return
      a_						   =     os.path.join(
          __UpperCAmelCase       ,							(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])

      if os.path.abspath(self.vocab_file) != os.path.abspath(__UpperCAmelCase):
         copyfile(self.vocab_file       ,							__UpperCAmelCase)

      return (out_vocab_file,)







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Optional[int]:
      a_						   =     list(conversation.iter_texts())
      if not all([is_user for is_user, msg in dialogue[::2]]) or not all(
          [not is_user for is_user, msg in dialogue[1::2]]):
         raise ValueError(
             "The model only supports 'user' and 'assistant' roles, starting with user and alternating (u/a/u/a/u...)")

      a_						   =     []
      if len(conversation.past_user_inputs) > 0:
         if not conversation.past_user_inputs[0].startswith(__UpperCAmelCase) or E_SYS not in conversation.past_user_inputs[0]:
            a_						   =     (
                B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
            )
      elif not dialogue[0][1].startswith(__UpperCAmelCase) or E_SYS not in dialogue[0][1]:
         a_						   =     (dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

      dialog_tokens += sum(
          [
              [self.bos_token_id]
              + self.encode(
                  F'''{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} '''       ,							add_special_tokens=__UpperCAmelCase)
              + [self.eos_token_id]
              for prompt, answer in zip(dialogue[::2]       ,							dialogue[1::2])
          ]       ,							[]       ,							)
      if not (dialogue[-1][0]):
         raise ValueError(F'''Last message must be from user, got {dialogue[-1]["role"]}''')
      dialog_tokens += [self.bos_token_id] + self.encode(
          F'''{B_INST} {(dialogue[-1][1]).strip()} {E_INST}'''       ,							add_special_tokens=__UpperCAmelCase)
      return dialog_tokens

"""simple docstring"""






import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 							snake_case       (     SCREAMING_SNAKE_CASE_							,       unittest.TestCase		):
   a_					:		Optional[int]     					=     GPTNeoXJapaneseTokenizer
   a_					:		Dict     					=     False
   a_					:		Union[str, Any]     					=     {"""do_clean_text""": False, """add_prefix_space""": False}







   def    UpperCAmelCase__	( self)   ->Union[str, Any]:
      super().setUp()

      a_						   =     [
          "こん",
          "こんに",
          "にちは",
          "ばんは",
          "世界,㔺界",
          "、",
          "。",
          "<BR>",
          "<SP>",
          "<TAB>",
          "<URL>",
          "<EMAIL>",
          "<TEL>",
          "<DATE>",
          "<PRICE>",
          "<BLOCK>",
          "<KIGOU>",
          "<U2000U2BFF>",
          "<|emoji1|>",
          "<unk>",
          "<|startoftext|>",
          "<|endoftext|>",
      ]
      a_						   =     {"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
      a_						   =     {"unk_token": "<unk>"}

      a_						   =     os.path.join(self.tmpdirname       ,							VOCAB_FILES_NAMES["vocab_file"])
      a_						   =     os.path.join(self.tmpdirname       ,							VOCAB_FILES_NAMES["emoji_file"])
      with open(self.vocab_file       ,							"w"       ,							encoding="utf-8") as vocab_writer:
         vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
      with open(self.emoji_file       ,							"w") as emoji_writer:
         emoji_writer.write(json.dumps(__UpperCAmelCase))







   def    UpperCAmelCase__	( self       ,							**__UpperCAmelCase)   ->int:
      kwargs.update(self.special_tokens_map)
      return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname       ,							**__UpperCAmelCase)







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Dict:
      a_						   =     "こんにちは、世界。 \nこんばんは、㔺界。😀"
      a_						   =     "こんにちは、世界。 \nこんばんは、世界。😀"
      return input_text, output_text







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Optional[Any]:
      a_   ,			a_						   =     self.get_input_output_texts(__UpperCAmelCase)
      a_						   =     tokenizer.encode(__UpperCAmelCase       ,							add_special_tokens=__UpperCAmelCase)
      a_						   =     tokenizer.decode(__UpperCAmelCase       ,							clean_up_tokenization_spaces=__UpperCAmelCase)
      return text, ids







   def    UpperCAmelCase__	( self)   ->Any:
      pass  # TODO add if relevant







   def    UpperCAmelCase__	( self)   ->int:
      pass  # TODO add if relevant







   def    UpperCAmelCase__	( self)   ->Optional[Any]:
      pass  # TODO add if relevant







   def    UpperCAmelCase__	( self)   ->List[str]:
      a_						   =     self.get_tokenizer()

      # Testing tokenization
      a_						   =     "こんにちは、世界。　こんばんは、㔺界。"
      a_						   =     ["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
      a_						   =     tokenizer.tokenize(__UpperCAmelCase)
      self.assertListEqual(__UpperCAmelCase       ,							__UpperCAmelCase)

      # Testing conversion to ids without special tokens
      a_						   =     [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
      a_						   =     tokenizer.convert_tokens_to_ids(__UpperCAmelCase)
      self.assertListEqual(__UpperCAmelCase       ,							__UpperCAmelCase)

      # Testing conversion to ids with special tokens
      a_						   =     tokens + [tokenizer.unk_token]
      a_						   =     [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
      a_						   =     tokenizer.convert_tokens_to_ids(__UpperCAmelCase)
      self.assertListEqual(__UpperCAmelCase       ,							__UpperCAmelCase)







   @slow
   def    UpperCAmelCase__	( self)   ->Tuple:
      a_						   =     self.tokenizer_class.from_pretrained("abeja/gpt-neox-japanese-2.7b")

      a_						   =     tokenizer.encode("ありがとう。"       ,							add_special_tokens=__UpperCAmelCase)
      a_						   =     tokenizer.encode("どういたしまして。"       ,							add_special_tokens=__UpperCAmelCase)

      a_						   =     tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase)
      a_						   =     tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase       ,							__UpperCAmelCase)

      assert encoded_sentence == ids_a
      assert encoded_pair == ids_a + ids_a







   def    UpperCAmelCase__	( self)   ->Tuple:
      # Intentionally convert some words to accommodate character fluctuations unique to Japanese
      pass







   def    UpperCAmelCase__	( self)   ->Union[str, Any]:
      # tokenizer has no padding token
      pass