


import uuid
from typing import Any, Dict, List, Optional, Union

from ..utils import add_end_docstrings, is_tf_available, is_torch_available, logging
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_tf_available():
   import tensorflow as tf

if is_torch_available():
   import torch


UpperCAmelCase_      : Tuple  					=       logging.get_logger(__name__)
class  SCREAMING_SNAKE_CASE__  :
      def __init__(       self				:       List[str]     ,							SCREAMING_SNAKE_CASE__				:       str = None     ,							SCREAMING_SNAKE_CASE__				:       uuid.UUID = None     ,							SCREAMING_SNAKE_CASE__				:       List[Any]=None     ,							SCREAMING_SNAKE_CASE__				:       int=None     )		->			List[Any]:
             if not conversation_id:
                    a_					:		Any							=  uuid.uuida()
             if past_user_inputs is None:
                    a_					:		int							=  []
             if generated_responses is None:
                    a_					:		int							=  []

             a_					:		uuid.UUID							=  conversation_id
             a_					:		List[str]							=  past_user_inputs
             a_					:		List[str]							=  generated_responses
             a_					:		Optional[str]							=  text
      def __eq__(       self				:       Any     ,							SCREAMING_SNAKE_CASE__				:       str     )		->			Tuple:
             if not isinstance(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     ):
                    return False
             if self.uuid == other.uuid:
                    return True
             return (
                 self.new_user_input == other.new_user_input
                 and self.past_user_inputs == other.past_user_inputs
                 and self.generated_responses == other.generated_responses
             )
      def        SCREAMING_SNAKE_CASE							(       self				:       Optional[int]     ,							SCREAMING_SNAKE_CASE__				:       str     ,							SCREAMING_SNAKE_CASE__				:       bool = False     )		->			Optional[Any]:
             if self.new_user_input:
                    if overwrite:
                           logger.warning(
                               F"""User input added while unprocessed input was existing: \"{self.new_user_input}\" was overwritten """
                               F"""with: \"{text}\"."""     )
                           a_					:		Optional[int]							=  text
                    else:
                           logger.warning(
                               F"""User input added while unprocessed input was existing: \"{self.new_user_input}\" new input """
                               F"""ignored: \"{text}\". Set `overwrite` to True to overwrite unprocessed user input"""     )
             else:
                    a_					:		Optional[Any]							=  text
      def        SCREAMING_SNAKE_CASE							(       self				:       List[Any]     )		->			List[str]:
             if self.new_user_input:
                    self.past_user_inputs.append(self.new_user_input     )
             a_					:		List[Any]							=  None
      def        SCREAMING_SNAKE_CASE							(       self				:       str     ,							SCREAMING_SNAKE_CASE__				:       str     )		->			Dict:
             self.generated_responses.append(SCREAMING_SNAKE_CASE__     )
      def        SCREAMING_SNAKE_CASE							(       self				:       int     )		->			Optional[Any]:
             for user_input, generated_response in zip(self.past_user_inputs     ,							self.generated_responses     ):
                    yield True, user_input
                    yield False, generated_response
             if self.new_user_input:
                    yield True, self.new_user_input

      def __repr__(       self				:       Tuple     )		->			Dict:
             a_					:		str							=  F"""Conversation id: {self.uuid} \n"""
             for is_user, text in self.iter_texts():
                    a_					:		List[str]							=  'user' if is_user else 'bot'
                    output += F"""{name} >> {text} \n"""
             return output
@add_end_docstrings(
    lowercase__						,    r'''
        min_length_for_response (`int`, *optional*, defaults to 32):
            The minimum length (in number of tokens) for a response.
        minimum_tokens (`int`, *optional*, defaults to 10):
            The minimum length of tokens to leave for a response.
    '''						,    )
class  SCREAMING_SNAKE_CASE__  (				lowercase__			):
      def __init__(       self				:       List[Any]     ,							*SCREAMING_SNAKE_CASE__				:       Any     ,							**SCREAMING_SNAKE_CASE__				:       Optional[int]     )		->			Optional[Any]:
             super().__init__(*SCREAMING_SNAKE_CASE__     ,							**SCREAMING_SNAKE_CASE__     )
             if self.tokenizer.pad_token_id is None:
                    a_					:		Dict							=  self.tokenizer.eos_token
      def        SCREAMING_SNAKE_CASE							(       self				:       Any     ,							SCREAMING_SNAKE_CASE__				:       List[Any]=None     ,							SCREAMING_SNAKE_CASE__				:       str=None     ,							SCREAMING_SNAKE_CASE__				:       Optional[int]=None     ,							**SCREAMING_SNAKE_CASE__				:       str     )		->			Union[str, Any]:
             a_					:		Optional[int]							=  {}
             a_					:		Optional[Any]							=  {}
             a_					:		Dict							=  {}

             if min_length_for_response is not None:
                    a_					:		Optional[int]							=  min_length_for_response
             if minimum_tokens is not None:
                    a_					:		Union[str, Any]							=  minimum_tokens

             if "max_length" in generate_kwargs:
                    a_					:		Any							=  generate_kwargs['max_length']
                    # self.max_length = generate_kwargs.get("max_length", self.model.config.max_length)
             if clean_up_tokenization_spaces is not None:
                    a_					:		List[Any]							=  clean_up_tokenization_spaces

             if generate_kwargs:
                    forward_params.update(SCREAMING_SNAKE_CASE__     )
             return preprocess_params, forward_params, postprocess_params
      def __call__(       self				:       Dict     ,							SCREAMING_SNAKE_CASE__				:       Union[Conversation, List[Conversation]]     ,							SCREAMING_SNAKE_CASE__				:       Any=0     ,							**SCREAMING_SNAKE_CASE__				:       List[str]     )		->			List[str]:
             a_					:		Dict							=  super().__call__(SCREAMING_SNAKE_CASE__     ,							num_workers=SCREAMING_SNAKE_CASE__     ,							**SCREAMING_SNAKE_CASE__     )
             if isinstance(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     ) and len(SCREAMING_SNAKE_CASE__     ) == 1:
                    return outputs[0]
             return outputs
      def        SCREAMING_SNAKE_CASE							(       self				:       Optional[Any]     ,							SCREAMING_SNAKE_CASE__				:       Conversation     ,							SCREAMING_SNAKE_CASE__				:       Optional[int]=3_2     )		->			Dict[str, Any]:
             if not isinstance(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     ):
                    raise ValueError('ConversationalPipeline, expects Conversation as inputs'     )
             if conversation.new_user_input is None:
                    raise ValueError(
                        F"""Conversation with UUID {type(conversation.uuid     )} does not contain new user input to process. """
                        'Add user inputs with the conversation\'s `add_user_input` method'     )
             if hasattr(self.tokenizer     ,							'_build_conversation_input_ids'     ):
                    a_					:		str							=  self.tokenizer._build_conversation_input_ids(SCREAMING_SNAKE_CASE__     )
             else:
                    # If the tokenizer cannot handle conversations, we default to only the old version
                    a_					:		Any							=  self._legacy_parse_and_tokenize(SCREAMING_SNAKE_CASE__     )

             if self.framework == "pt":
                    a_					:		Any							=  torch.LongTensor([input_ids]     )
             elif self.framework == "tf":
                    a_					:		Dict							=  tf.constant([input_ids]     )
             return {"input_ids": input_ids, "conversation": conversation}
      def        SCREAMING_SNAKE_CASE							(       self				:       int     ,							SCREAMING_SNAKE_CASE__				:       Tuple     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]=1_0     ,							**SCREAMING_SNAKE_CASE__				:       Optional[Any]     )		->			Any:
             a_					:		Union[str, Any]							=  generate_kwargs.get('max_length'     ,							self.model.config.max_length     )

             a_					:		int							=  model_inputs['input_ids'].shape[1]
             if max_length - minimum_tokens < n:
                    logger.warning(F"""Conversation input is to long ({n}), trimming it to ({max_length} - {minimum_tokens})"""     )
                    a_					:		Any							=  max_length - minimum_tokens
                    a_					:		Tuple							=  model_inputs['input_ids'][:, -trim:]
                    if "attention_mask" in model_inputs:
                           a_					:		Tuple							=  model_inputs['attention_mask'][:, -trim:]
             a_					:		Dict							=  model_inputs.pop('conversation'     )
             a_					:		List[Any]							=  max_length
             a_					:		Optional[int]							=  self.model.generate(**SCREAMING_SNAKE_CASE__     ,							**SCREAMING_SNAKE_CASE__     )
             if self.model.config.is_encoder_decoder:
                    a_					:		Optional[int]							=  1
             else:
                    a_					:		List[str]							=  n
             return {"output_ids": output_ids[:, start_position:], "conversation": conversation}
      def        SCREAMING_SNAKE_CASE							(       self				:       Tuple     ,							SCREAMING_SNAKE_CASE__				:       Tuple     ,							SCREAMING_SNAKE_CASE__				:       Dict=True     )		->			Optional[Any]:
             a_					:		Optional[int]							=  model_outputs['output_ids']
             a_					:		List[Any]							=  self.tokenizer.decode(
                 output_ids[0]     ,							skip_special_tokens=SCREAMING_SNAKE_CASE__     ,							clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE__     ,							)
             a_					:		Optional[int]							=  model_outputs['conversation']
             conversation.mark_processed()
             conversation.append_response(SCREAMING_SNAKE_CASE__     )
             return conversation

      def        SCREAMING_SNAKE_CASE							(       self				:       Optional[int]     ,							SCREAMING_SNAKE_CASE__				:       Conversation     )		->			Dict:
             a_					:		Optional[int]							=  self.tokenizer.eos_token_id
             a_					:		List[Any]							=  []
             for is_user, text in conversation.iter_texts():
                    if eos_token_id is not None:
                           input_ids.extend(self.tokenizer.encode(SCREAMING_SNAKE_CASE__     ,							add_special_tokens=SCREAMING_SNAKE_CASE__     ) + [eos_token_id]     )
                    else:
                           input_ids.extend(self.tokenizer.encode(SCREAMING_SNAKE_CASE__     ,							add_special_tokens=SCREAMING_SNAKE_CASE__     )     )

             if len(SCREAMING_SNAKE_CASE__     ) > self.tokenizer.model_max_length:
                    a_					:		str							=  input_ids[-self.tokenizer.model_max_length :]
             return input_ids






import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class  SCREAMING_SNAKE_CASE__  (				lowercase__						,    unittest.TestCase			):
      snake_case__				:     Any        =						GPTSanJapaneseTokenizer
      snake_case__				:     Tuple        =						False
      snake_case__				:     str        =						{'''do_clean_text''': False, '''add_prefix_space''': False}
      def        SCREAMING_SNAKE_CASE							(       self				:       List[Any]     )		->			str:
             super().setUp()

             # fmt: off
             a_					:		Union[str, Any]							=  ['こん', 'こんに', 'にちは', 'ばんは', '世界,㔺界', '、', '。', '<BR>', '<SP>', '<TAB>', '<URL>', '<EMAIL>', '<TEL>', '<DATE>', '<PRICE>', '<BLOCK>', '<KIGOU>', '<U2000U2BFF>', '<|emoji1|>', '<unk>', '<|bagoftoken|>', '<|endoftext|>']
             # fmt: on
             a_					:		int							=  {'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
             a_					:		List[Any]							=  {'unk_token': '<unk>'}

             a_					:		Any							=  os.path.join(self.tmpdirname     ,							VOCAB_FILES_NAMES['vocab_file']     )
             a_					:		List[str]							=  os.path.join(self.tmpdirname     ,							VOCAB_FILES_NAMES['emoji_file']     )
             with open(self.vocab_file     ,							'w'     ,							encoding='utf-8'     ) as vocab_writer:
                    vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )
             with open(self.emoji_file     ,							'w'     ) as emoji_writer:
                    emoji_writer.write(json.dumps(SCREAMING_SNAKE_CASE__     )     )
      def        SCREAMING_SNAKE_CASE							(       self				:       List[str]     ,							**SCREAMING_SNAKE_CASE__				:       Tuple     )		->			int:
             kwargs.update(self.special_tokens_map     )
             return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname     ,							**SCREAMING_SNAKE_CASE__     )
      def        SCREAMING_SNAKE_CASE							(       self				:       Dict     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]     )		->			int:
             a_					:		Optional[int]							=  'こんにちは、世界。 \nこんばんは、㔺界。😀'
             a_					:		List[str]							=  'こんにちは、世界。 \nこんばんは、世界。😀'
             return input_text, output_text
      def        SCREAMING_SNAKE_CASE							(       self				:       str     ,							SCREAMING_SNAKE_CASE__				:       int     )		->			Dict:
             a_							,		a_					:		Union[str, Any]							=  self.get_input_output_texts(SCREAMING_SNAKE_CASE__     )
             a_					:		List[Any]							=  tokenizer.encode(SCREAMING_SNAKE_CASE__     ,							add_special_tokens=SCREAMING_SNAKE_CASE__     )
             a_					:		Dict							=  tokenizer.decode(SCREAMING_SNAKE_CASE__     ,							clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE__     )
             return text, ids
      def        SCREAMING_SNAKE_CASE							(       self				:       Dict     )		->			int:
             pass  # TODO add if relevant
      def        SCREAMING_SNAKE_CASE							(       self				:       Dict     )		->			Any:
             pass  # TODO add if relevant
      def        SCREAMING_SNAKE_CASE							(       self				:       Any     )		->			Tuple:
             pass  # TODO add if relevant
      def        SCREAMING_SNAKE_CASE							(       self				:       Dict     )		->			List[str]:
             a_					:		List[str]							=  self.get_tokenizer()

             # Testing tokenization
             a_					:		List[Any]							=  'こんにちは、世界。　こんばんは、㔺界。'
             a_					:		Optional[int]							=  ['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
             a_					:		Dict							=  tokenizer.tokenize(SCREAMING_SNAKE_CASE__     )
             self.assertListEqual(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     )

             # Testing conversion to ids without special tokens
             a_					:		Tuple							=  [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
             a_					:		List[Any]							=  tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__     )
             self.assertListEqual(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     )

             # Testing conversion to ids with special tokens
             a_					:		int							=  tokens + [tokenizer.unk_token]
             a_					:		int							=  [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 1_9]
             a_					:		Tuple							=  tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__     )
             self.assertListEqual(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     )
      def        SCREAMING_SNAKE_CASE							(       self				:       List[str]     )		->			Dict:
             a_					:		Union[str, Any]							=  self.get_tokenizer()

             # Testing tokenization
             a_					:		Dict							=  'こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'
             a_					:		List[Any]							=  'こんにちは、、、、世界。こんばんは、、、、世界。'
             a_					:		Any							=  tokenizer.encode(SCREAMING_SNAKE_CASE__     )
             a_					:		Optional[Any]							=  tokenizer.decode(SCREAMING_SNAKE_CASE__     )
             self.assertEqual(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     )
      @slow
      def        SCREAMING_SNAKE_CASE							(       self				:       str     )		->			Dict:
             a_					:		Tuple							=  self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'     )

             # Testing tokenization
             a_					:		List[Any]							=  'こんにちは、世界。'
             a_					:		int							=  'こんばんは、㔺界。😀'
             a_					:		Dict							=  'こんにちは、世界。こんばんは、世界。😀'
             a_					:		Optional[int]							=  tokenizer.encode(prefix_text + input_text     )
             a_					:		Any							=  tokenizer.encode(''     ,							prefix_text=prefix_text + input_text     )
             a_					:		Union[str, Any]							=  tokenizer.encode(SCREAMING_SNAKE_CASE__     ,							prefix_text=SCREAMING_SNAKE_CASE__     )
             a_					:		List[Any]							=  tokenizer.decode(SCREAMING_SNAKE_CASE__     )
             a_					:		Tuple							=  tokenizer.decode(SCREAMING_SNAKE_CASE__     )
             a_					:		str							=  tokenizer.decode(SCREAMING_SNAKE_CASE__     )
             self.assertEqual(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     )
             self.assertEqual(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     )
             self.assertEqual(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     )
      @slow
      def        SCREAMING_SNAKE_CASE							(       self				:       Optional[int]     )		->			Optional[int]:
             a_					:		Tuple							=  self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'     )

             # Testing tokenization
             a_					:		str							=  'こんにちは、世界。'
             a_					:		List[str]							=  'こんばんは、㔺界。😀'

             a_					:		str							=  len(tokenizer.encode(SCREAMING_SNAKE_CASE__     )     ) - 2
             a_					:		Tuple							=  len(tokenizer.encode(SCREAMING_SNAKE_CASE__     )     ) - 2

             a_					:		Optional[Any]							=  [1] + [0] * (len_prefix + len_text + 1)
             a_					:		Optional[Any]							=  [1] * (len_prefix + len_text + 1) + [0]
             a_					:		Tuple							=  [1] + [1] * (len_prefix) + [0] * (len_text + 1)

             a_					:		List[str]							=  tokenizer(prefix_text + input_text     ).token_type_ids
             a_					:		Union[str, Any]							=  tokenizer(''     ,							prefix_text=prefix_text + input_text     ).token_type_ids
             a_					:		Any							=  tokenizer(SCREAMING_SNAKE_CASE__     ,							prefix_text=SCREAMING_SNAKE_CASE__     ).token_type_ids
             self.assertListEqual(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     )
             self.assertListEqual(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     )
             self.assertListEqual(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     )
      @slow
      def        SCREAMING_SNAKE_CASE							(       self				:       Union[str, Any]     )		->			int:
             a_					:		str							=  self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'     )

             a_					:		Optional[int]							=  tokenizer.encode('あンいワ'     )
             a_					:		Dict							=  tokenizer.encode(''     ,							prefix_text='あンいワ'     )
             a_					:		Dict							=  tokenizer.encode('いワ'     ,							prefix_text='あン'     )

             self.assertEqual(tokenizer.decode(SCREAMING_SNAKE_CASE__     )     ,							tokenizer.decode(SCREAMING_SNAKE_CASE__     )     )
             self.assertEqual(tokenizer.decode(SCREAMING_SNAKE_CASE__     )     ,							tokenizer.decode(SCREAMING_SNAKE_CASE__     )     )
             self.assertNotEqual(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     )
             self.assertNotEqual(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     )
             self.assertEqual(x_token_a[1]     ,							x_token_a[-1]     )  # SEG token
             self.assertEqual(x_token_a[1]     ,							x_token_a[3]     )  # SEG token
      @slow
      def        SCREAMING_SNAKE_CASE							(       self				:       Dict     )		->			Optional[Any]:
             a_					:		List[str]							=  self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'     )

             a_					:		Optional[Any]							=  [['武田信玄', 'は、'], ['織田信長', 'の配下の、']]
             a_					:		List[str]							=  tokenizer(SCREAMING_SNAKE_CASE__     ,							padding=SCREAMING_SNAKE_CASE__     )
             a_					:		Dict							=  tokenizer.batch_encode_plus(SCREAMING_SNAKE_CASE__     ,							padding=SCREAMING_SNAKE_CASE__     )

             # fmt: off
             a_					:		List[Any]							=  [[3_5_9_9_3, 8_6_4_0, 2_5_9_4_8, 3_5_9_9_8, 3_0_6_4_7, 3_5_6_7_5, 3_5_9_9_9, 3_5_9_9_9], [3_5_9_9_3, 1_0_3_8_2, 9_8_6_8, 3_5_9_9_8, 3_0_6_4_6, 9_4_5_9, 3_0_6_4_6, 3_5_6_7_5]]
             a_					:		Any							=  [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
             a_					:		List[Any]							=  [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
             # fmt: on
             self.assertListEqual(x_token.input_ids     ,							SCREAMING_SNAKE_CASE__     )
             self.assertListEqual(x_token.token_type_ids     ,							SCREAMING_SNAKE_CASE__     )
             self.assertListEqual(x_token.attention_mask     ,							SCREAMING_SNAKE_CASE__     )
             self.assertListEqual(x_token_a.input_ids     ,							SCREAMING_SNAKE_CASE__     )
             self.assertListEqual(x_token_a.token_type_ids     ,							SCREAMING_SNAKE_CASE__     )
             self.assertListEqual(x_token_a.attention_mask     ,							SCREAMING_SNAKE_CASE__     )
      def        SCREAMING_SNAKE_CASE							(       self				:       Tuple     )		->			Dict:
             # Intentionally convert some words to accommodate character fluctuations unique to Japanese
             pass

      def        SCREAMING_SNAKE_CASE							(       self				:       Dict     )		->			int:
             # tokenizer has no padding token
             pass


