





import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 					A__   (       __snake_case       ,   unittest.TestCase			):
      _UpperCAmelCase	:Any						    =   GPTSanJapaneseTokenizer
      _UpperCAmelCase	:Optional[int]						    =   False
      _UpperCAmelCase	:Dict						    =   {'do_clean_text': False, 'add_prefix_space': False}





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       super().setUp()

       # fmt: off
       UpperCamelCase	:					Optional[int]       	=   ["こん", "こんに", "にちは", "ばんは", "世界,㔺界", "、", "。", "<BR>", "<SP>", "<TAB>", "<URL>", "<EMAIL>", "<TEL>", "<DATE>", "<PRICE>", "<BLOCK>", "<KIGOU>", "<U2000U2BFF>", "<|emoji1|>", "<unk>", "<|bagoftoken|>", "<|endoftext|>"]
       # fmt: on
       UpperCamelCase	:					Dict       	=   {"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
       UpperCamelCase	:					Any       	=   {"unk_token": "<unk>"}

       UpperCamelCase	:					Any       	=   os.path.join(self.tmpdirname       ,							VOCAB_FILES_NAMES["vocab_file"]   )
       UpperCamelCase	:					Union[str, Any]       	=   os.path.join(self.tmpdirname       ,							VOCAB_FILES_NAMES["emoji_file"]   )
       with open(self.vocab_file       ,							"w"       ,							encoding="utf-8"   ) as vocab_writer:
        vocab_writer.write("".join([x + "\n" for x in vocab_tokens]   )   )
       with open(self.emoji_file       ,							"w"   ) as emoji_writer:
        emoji_writer.write(json.dumps(A_   )   )





      def        __UpperCamelCase( self       ,							**A_   ):
       '''simple docstring'''




       kwargs.update(self.special_tokens_map   )
       return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname       ,							**A_   )





      def        __UpperCamelCase( self       ,							A_   ):
       '''simple docstring'''




       UpperCamelCase	:					Tuple       	=   "こんにちは、世界。 \nこんばんは、㔺界。😀"
       UpperCamelCase	:					Optional[Any]       	=   "こんにちは、世界。 \nこんばんは、世界。😀"
       return input_text, output_text





      def        __UpperCamelCase( self       ,							A_   ):
       '''simple docstring'''




       UpperCamelCase	,    UpperCamelCase	:					Tuple       	=   self.get_input_output_texts(A_   )
       UpperCamelCase	:					List[Any]       	=   tokenizer.encode(A_       ,							add_special_tokens=A_   )
       UpperCamelCase	:					Any       	=   tokenizer.decode(A_       ,							clean_up_tokenization_spaces=A_   )
       return text, ids





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       pass  # TODO add if relevant





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       pass  # TODO add if relevant





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       pass  # TODO add if relevant





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       UpperCamelCase	:					Any       	=   self.get_tokenizer()

       # Testing tokenization
       UpperCamelCase	:					str       	=   "こんにちは、世界。　こんばんは、㔺界。"
       UpperCamelCase	:					List[str]       	=   ["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
       UpperCamelCase	:					int       	=   tokenizer.tokenize(A_   )
       self.assertListEqual(A_       ,							A_   )

       # Testing conversion to ids without special tokens
       UpperCamelCase	:					Tuple       	=   [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
       UpperCamelCase	:					Any       	=   tokenizer.convert_tokens_to_ids(A_   )
       self.assertListEqual(A_       ,							A_   )

       # Testing conversion to ids with special tokens
       UpperCamelCase	:					str       	=   tokens + [tokenizer.unk_token]
       UpperCamelCase	:					int       	=   [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
       UpperCamelCase	:					Any       	=   tokenizer.convert_tokens_to_ids(A_   )
       self.assertListEqual(A_       ,							A_   )





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       UpperCamelCase	:					Union[str, Any]       	=   self.get_tokenizer()

       # Testing tokenization
       UpperCamelCase	:					int       	=   "こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"
       UpperCamelCase	:					Optional[int]       	=   "こんにちは、、、、世界。こんばんは、、、、世界。"
       UpperCamelCase	:					Dict       	=   tokenizer.encode(A_   )
       UpperCamelCase	:					Dict       	=   tokenizer.decode(A_   )
       self.assertEqual(A_       ,							A_   )





      @slow
      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       UpperCamelCase	:					List[Any]       	=   self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"   )

       # Testing tokenization
       UpperCamelCase	:					List[str]       	=   "こんにちは、世界。"
       UpperCamelCase	:					int       	=   "こんばんは、㔺界。😀"
       UpperCamelCase	:					List[str]       	=   "こんにちは、世界。こんばんは、世界。😀"
       UpperCamelCase	:					Any       	=   tokenizer.encode(prefix_text + input_text   )
       UpperCamelCase	:					List[Any]       	=   tokenizer.encode(""       ,							prefix_text=prefix_text + input_text   )
       UpperCamelCase	:					Any       	=   tokenizer.encode(A_       ,							prefix_text=A_   )
       UpperCamelCase	:					Any       	=   tokenizer.decode(A_   )
       UpperCamelCase	:					Any       	=   tokenizer.decode(A_   )
       UpperCamelCase	:					Tuple       	=   tokenizer.decode(A_   )
       self.assertEqual(A_       ,							A_   )
       self.assertEqual(A_       ,							A_   )
       self.assertEqual(A_       ,							A_   )





      @slow
      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       UpperCamelCase	:					Optional[int]       	=   self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"   )

       # Testing tokenization
       UpperCamelCase	:					Union[str, Any]       	=   "こんにちは、世界。"
       UpperCamelCase	:					Optional[Any]       	=   "こんばんは、㔺界。😀"

       UpperCamelCase	:					Any       	=   len(tokenizer.encode(A_   )   ) - 2
       UpperCamelCase	:					str       	=   len(tokenizer.encode(A_   )   ) - 2

       UpperCamelCase	:					Optional[int]       	=   [1] + [0] * (len_prefix + len_text + 1)
       UpperCamelCase	:					Union[str, Any]       	=   [1] * (len_prefix + len_text + 1) + [0]
       UpperCamelCase	:					Dict       	=   [1] + [1] * (len_prefix) + [0] * (len_text + 1)

       UpperCamelCase	:					Union[str, Any]       	=   tokenizer(prefix_text + input_text   ).token_type_ids
       UpperCamelCase	:					Optional[Any]       	=   tokenizer(""       ,							prefix_text=prefix_text + input_text   ).token_type_ids
       UpperCamelCase	:					str       	=   tokenizer(A_       ,							prefix_text=A_   ).token_type_ids
       self.assertListEqual(A_       ,							A_   )
       self.assertListEqual(A_       ,							A_   )
       self.assertListEqual(A_       ,							A_   )





      @slow
      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       UpperCamelCase	:					Any       	=   self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"   )

       UpperCamelCase	:					Optional[int]       	=   tokenizer.encode("あンいワ"   )
       UpperCamelCase	:					Union[str, Any]       	=   tokenizer.encode(""       ,							prefix_text="あンいワ"   )
       UpperCamelCase	:					str       	=   tokenizer.encode("いワ"       ,							prefix_text="あン"   )

       self.assertEqual(tokenizer.decode(A_   )       ,							tokenizer.decode(A_   )   )
       self.assertEqual(tokenizer.decode(A_   )       ,							tokenizer.decode(A_   )   )
       self.assertNotEqual(A_       ,							A_   )
       self.assertNotEqual(A_       ,							A_   )
       self.assertEqual(x_token_a[1]       ,							x_token_a[-1]   )  # SEG token
       self.assertEqual(x_token_a[1]       ,							x_token_a[3]   )  # SEG token





      @slow
      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       UpperCamelCase	:					List[Any]       	=   self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"   )

       UpperCamelCase	:					Dict       	=   [["武田信玄", "は、"], ["織田信長", "の配下の、"]]
       UpperCamelCase	:					str       	=   tokenizer(A_       ,							padding=A_   )
       UpperCamelCase	:					List[str]       	=   tokenizer.batch_encode_plus(A_       ,							padding=A_   )

       # fmt: off
       UpperCamelCase	:					List[str]       	=   [[3_5993, 8640, 2_5948, 3_5998, 3_0647, 3_5675, 3_5999, 3_5999], [3_5993, 1_0382, 9868, 3_5998, 3_0646, 9459, 3_0646, 3_5675]]
       UpperCamelCase	:					int       	=   [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
       UpperCamelCase	:					int       	=   [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
       # fmt: on
       self.assertListEqual(x_token.input_ids       ,							A_   )
       self.assertListEqual(x_token.token_type_ids       ,							A_   )
       self.assertListEqual(x_token.attention_mask       ,							A_   )
       self.assertListEqual(x_token_a.input_ids       ,							A_   )
       self.assertListEqual(x_token_a.token_type_ids       ,							A_   )
       self.assertListEqual(x_token_a.attention_mask       ,							A_   )





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       pass





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       pass







class 					A__   :





      def __init__( self       ,							A_   ):
       '''simple docstring'''




       UpperCamelCase	:					Union[str, Any]       	=   set_counts
       UpperCamelCase	:					int       	=   max(A_   )
       UpperCamelCase	:					Optional[Any]       	=   len(A_   )
       UpperCamelCase	:					Union[str, Any]       	=   [1] * num_sets
       UpperCamelCase	:					Union[str, Any]       	=   list(range(A_   )   )





      def        __UpperCamelCase( self       ,							A_       ,							A_   ):
       '''simple docstring'''




       UpperCamelCase	:					Any       	=   self.get_parent(A_   )
       UpperCamelCase	:					Optional[int]       	=   self.get_parent(A_   )

       if src_parent == dst_parent:
        return False

       if self.ranks[dst_parent] >= self.ranks[src_parent]:
        self.set_counts[dst_parent] += self.set_counts[src_parent]
        UpperCamelCase	:					int       	=   0
        UpperCamelCase	:					Dict       	=   dst_parent
        if self.ranks[dst_parent] == self.ranks[src_parent]:
         self.ranks[dst_parent] += 1
        UpperCamelCase	:					Optional[int]       	=   self.set_counts[dst_parent]
       else:
        self.set_counts[src_parent] += self.set_counts[dst_parent]
        UpperCamelCase	:					Any       	=   0
        UpperCamelCase	:					Optional[int]       	=   src_parent
        UpperCamelCase	:					int       	=   self.set_counts[src_parent]

       UpperCamelCase	:					Any       	=   max(self.max_set       ,							A_   )
       return True





      def        __UpperCamelCase( self       ,							A_   ):
       '''simple docstring'''




       if self.parents[disj_set] == disj_set:
        return disj_set
       UpperCamelCase	:					Optional[int]       	=   self.get_parent(self.parents[disj_set]   )
       return self.parents[disj_set]
