

import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin



@custom_tokenizers
class   __lowercase				( UpperCAmelCase_		,       unittest.TestCase	):




			"""simple docstring"""







			_UpperCAmelCase   :							Optional[int]										=     BertJapaneseTokenizer
			_UpperCAmelCase   :							Any										=     False
			_UpperCAmelCase   :							Dict										=     True



			def        _SCREAMING_SNAKE_CASE			( self  :     Optional[int]):
									super().setUp()

									SCREAMING_SNAKE_CASE_: Dict  		=       [
									    "[UNK]",
									    "[CLS]",
									    "[SEP]",
									    "こんにちは",
									    "こん",
									    "にちは",
									    "ばんは",
									    "##こん",
									    "##にちは",
									    "##ばんは",
									    "世界",
									    "##世界",
									    "、",
									    "##、",
									    "。",
									    "##。",
									]

									SCREAMING_SNAKE_CASE_: Tuple  		=       os.path.join(self.tmpdirname   ,						VOCAB_FILES_NAMES["vocab_file"])
									with open(self.vocab_file   ,						"w"   ,						encoding="utf-8") as vocab_writer:
															vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))



			def        _SCREAMING_SNAKE_CASE			( self  :     Optional[int]   ,						lowerCAmelCase__  :     Optional[int]):
									SCREAMING_SNAKE_CASE_: Optional[int]  		=       "こんにちは、世界。 \nこんばんは、世界。"
									SCREAMING_SNAKE_CASE_: Dict  		=       "こんにちは 、 世界 。 こんばんは 、 世界 。"
									return input_text, output_text



			def        _SCREAMING_SNAKE_CASE			( self  :     List[str]   ,						lowerCAmelCase__  :     List[str]):
									SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Any  		=       self.get_input_output_texts(lowerCAmelCase__)
									SCREAMING_SNAKE_CASE_: Dict  		=       tokenizer.encode(lowerCAmelCase__   ,						add_special_tokens=lowerCAmelCase__)
									SCREAMING_SNAKE_CASE_: List[Any]  		=       tokenizer.decode(lowerCAmelCase__   ,						clean_up_tokenization_spaces=lowerCAmelCase__)
									return text, ids



			def        _SCREAMING_SNAKE_CASE			( self  :     List[Any]):
									pass  # TODO add if relevant



			def        _SCREAMING_SNAKE_CASE			( self  :     Dict):
									pass  # TODO add if relevant



			def        _SCREAMING_SNAKE_CASE			( self  :     Optional[int]):
									pass  # TODO add if relevant



			def        _SCREAMING_SNAKE_CASE			( self  :     int):
									SCREAMING_SNAKE_CASE_: int  		=       self.tokenizer_class(self.vocab_file)

									SCREAMING_SNAKE_CASE_: List[Any]  		=       tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。")
									self.assertListEqual(lowerCAmelCase__   ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"])
									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__)   ,						[3, 12, 10, 14, 4, 9, 12, 10, 14])



			def        _SCREAMING_SNAKE_CASE			( self  :     Optional[int]):
									SCREAMING_SNAKE_CASE_: int  		=       self.tokenizer_class(self.vocab_file   ,						word_tokenizer_type="mecab")
									self.assertIsNotNone(lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_: Dict  		=       "こんにちは、世界。\nこんばんは、世界。"
									SCREAMING_SNAKE_CASE_: Optional[int]  		=       tokenizer.tokenize(lowerCAmelCase__)
									self.assertListEqual(lowerCAmelCase__   ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"])
									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__)   ,						[3, 12, 10, 14, 4, 9, 12, 10, 14])

									SCREAMING_SNAKE_CASE_: int  		=       os.path.join(self.tmpdirname   ,						"tokenizer.bin")
									with open(lowerCAmelCase__   ,						"wb") as handle:
															pickle.dump(lowerCAmelCase__   ,						lowerCAmelCase__)

									with open(lowerCAmelCase__   ,						"rb") as handle:
															SCREAMING_SNAKE_CASE_: List[Any]  		=       pickle.load(lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_: Optional[Any]  		=       tokenizer_new.tokenize(lowerCAmelCase__)

									self.assertListEqual(lowerCAmelCase__   ,						lowerCAmelCase__)



			def        _SCREAMING_SNAKE_CASE			( self  :     Optional[Any]):
									SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       MecabTokenizer(mecab_dic="ipadic")

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,						["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]   ,						)



			def        _SCREAMING_SNAKE_CASE			( self  :     Optional[Any]):
									try:
															SCREAMING_SNAKE_CASE_: List[str]  		=       MecabTokenizer(mecab_dic="unidic_lite")
									except ModuleNotFoundError:
															return

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]   ,						)



			def        _SCREAMING_SNAKE_CASE			( self  :     str):
									try:
															SCREAMING_SNAKE_CASE_: Tuple  		=       MecabTokenizer(mecab_dic="unidic")
									except ModuleNotFoundError:
															return

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]   ,						)



			def        _SCREAMING_SNAKE_CASE			( self  :     Tuple):
									SCREAMING_SNAKE_CASE_: Tuple  		=       MecabTokenizer(do_lower_case=lowerCAmelCase__   ,						mecab_dic="ipadic")

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,						["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]   ,						)



			def        _SCREAMING_SNAKE_CASE			( self  :     int):
									try:
															SCREAMING_SNAKE_CASE_: int  		=       MecabTokenizer(
															    do_lower_case=lowerCAmelCase__   ,						normalize_text=lowerCAmelCase__   ,						mecab_option="-d /usr/local/lib/mecab/dic/jumandic")
									except RuntimeError:
															# if dict doesn't exist in the system, previous code raises this error.
															return

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,						["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]   ,						)



			def        _SCREAMING_SNAKE_CASE			( self  :     List[Any]):
									SCREAMING_SNAKE_CASE_: Dict  		=       MecabTokenizer(normalize_text=lowerCAmelCase__   ,						mecab_dic="ipadic")

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,						["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]   ,						)



			@require_sudachi
			def        _SCREAMING_SNAKE_CASE			( self  :     Dict):
									SCREAMING_SNAKE_CASE_: Any  		=       self.tokenizer_class(self.vocab_file   ,						word_tokenizer_type="sudachi")
									self.assertIsNotNone(lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_: List[str]  		=       "こんにちは、世界。\nこんばんは、世界。"
									SCREAMING_SNAKE_CASE_: Any  		=       tokenizer.tokenize(lowerCAmelCase__)
									self.assertListEqual(lowerCAmelCase__   ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"])
									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__)   ,						[3, 12, 10, 14, 4, 9, 12, 10, 14])

									SCREAMING_SNAKE_CASE_: Tuple  		=       os.path.join(self.tmpdirname   ,						"tokenizer.bin")
									with open(lowerCAmelCase__   ,						"wb") as handle:
															pickle.dump(lowerCAmelCase__   ,						lowerCAmelCase__)

									with open(lowerCAmelCase__   ,						"rb") as handle:
															SCREAMING_SNAKE_CASE_: str  		=       pickle.load(lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_: str  		=       tokenizer_new.tokenize(lowerCAmelCase__)

									self.assertListEqual(lowerCAmelCase__   ,						lowerCAmelCase__)



			@require_sudachi
			def        _SCREAMING_SNAKE_CASE			( self  :     List[str]):
									SCREAMING_SNAKE_CASE_: Optional[int]  		=       SudachiTokenizer(sudachi_dict_type="core")

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,						[" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]   ,						)



			@require_sudachi
			def        _SCREAMING_SNAKE_CASE			( self  :     Any):
									SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       SudachiTokenizer(sudachi_dict_type="core"   ,						sudachi_split_mode="A")

									self.assertListEqual(tokenizer.tokenize("外国人参政権")   ,						["外国", "人", "参政", "権"])



			@require_sudachi
			def        _SCREAMING_SNAKE_CASE			( self  :     Union[str, Any]):
									SCREAMING_SNAKE_CASE_: Tuple  		=       SudachiTokenizer(sudachi_dict_type="core"   ,						sudachi_split_mode="B")

									self.assertListEqual(tokenizer.tokenize("外国人参政権")   ,						["外国人", "参政権"])



			@require_sudachi
			def        _SCREAMING_SNAKE_CASE			( self  :     Optional[int]):
									SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       SudachiTokenizer(sudachi_dict_type="core"   ,						sudachi_split_mode="C")

									self.assertListEqual(tokenizer.tokenize("外国人参政権")   ,						["外国人参政権"])



			@require_sudachi
			def        _SCREAMING_SNAKE_CASE			( self  :     Dict):
									SCREAMING_SNAKE_CASE_: Optional[Any]  		=       SudachiTokenizer(do_lower_case=lowerCAmelCase__   ,						sudachi_dict_type="core")

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,						[" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]   ,						)



			@require_sudachi
			def        _SCREAMING_SNAKE_CASE			( self  :     Any):
									SCREAMING_SNAKE_CASE_: str  		=       SudachiTokenizer(normalize_text=lowerCAmelCase__   ,						sudachi_dict_type="core")

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,						[" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]   ,						)



			@require_sudachi
			def        _SCREAMING_SNAKE_CASE			( self  :     List[str]):
									SCREAMING_SNAKE_CASE_: Optional[Any]  		=       SudachiTokenizer(trim_whitespace=lowerCAmelCase__   ,						sudachi_dict_type="core")

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]   ,						)



			@require_jumanpp
			def        _SCREAMING_SNAKE_CASE			( self  :     Union[str, Any]):
									SCREAMING_SNAKE_CASE_: Tuple  		=       self.tokenizer_class(self.vocab_file   ,						word_tokenizer_type="jumanpp")
									self.assertIsNotNone(lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_: int  		=       "こんにちは、世界。\nこんばんは、世界。"
									SCREAMING_SNAKE_CASE_: Dict  		=       tokenizer.tokenize(lowerCAmelCase__)
									self.assertListEqual(lowerCAmelCase__   ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"])
									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__)   ,						[3, 12, 10, 14, 4, 9, 12, 10, 14])

									SCREAMING_SNAKE_CASE_: Optional[int]  		=       os.path.join(self.tmpdirname   ,						"tokenizer.bin")
									with open(lowerCAmelCase__   ,						"wb") as handle:
															pickle.dump(lowerCAmelCase__   ,						lowerCAmelCase__)

									with open(lowerCAmelCase__   ,						"rb") as handle:
															SCREAMING_SNAKE_CASE_: Dict  		=       pickle.load(lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_: Dict  		=       tokenizer_new.tokenize(lowerCAmelCase__)

									self.assertListEqual(lowerCAmelCase__   ,						lowerCAmelCase__)



			@require_jumanpp
			def        _SCREAMING_SNAKE_CASE			( self  :     Any):
									SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       JumanppTokenizer()

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,						["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]   ,						)



			@require_jumanpp
			def        _SCREAMING_SNAKE_CASE			( self  :     str):
									SCREAMING_SNAKE_CASE_: List[str]  		=       JumanppTokenizer(do_lower_case=lowerCAmelCase__)

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,						["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]   ,						)



			@require_jumanpp
			def        _SCREAMING_SNAKE_CASE			( self  :     Tuple):
									SCREAMING_SNAKE_CASE_: Dict  		=       JumanppTokenizer(normalize_text=lowerCAmelCase__)

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,						["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]   ,						)



			@require_jumanpp
			def        _SCREAMING_SNAKE_CASE			( self  :     Dict):
									SCREAMING_SNAKE_CASE_: str  		=       JumanppTokenizer(trim_whitespace=lowerCAmelCase__)

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]   ,						)



			@require_jumanpp
			def        _SCREAMING_SNAKE_CASE			( self  :     List[Any]):
									SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       JumanppTokenizer()

									self.assertListEqual(
									    tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。")   ,						["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]   ,						)



			def        _SCREAMING_SNAKE_CASE			( self  :     List[str]):
									SCREAMING_SNAKE_CASE_: Optional[Any]  		=       ["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

									SCREAMING_SNAKE_CASE_: Optional[int]  		=       {}
									for i, token in enumerate(lowerCAmelCase__):
															SCREAMING_SNAKE_CASE_: Optional[Any]  		=       i
									SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       WordpieceTokenizer(vocab=lowerCAmelCase__   ,						unk_token="[UNK]")

									self.assertListEqual(tokenizer.tokenize("")   ,						[])

									self.assertListEqual(tokenizer.tokenize("こんにちは")   ,						["こんにちは"])

									self.assertListEqual(tokenizer.tokenize("こんばんは")   ,						["こん", "##ばんは"])

									self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは")   ,						["こん", "##ばんは", "[UNK]", "こんにちは"])



			def        _SCREAMING_SNAKE_CASE			( self  :     Optional[Any]):
									SCREAMING_SNAKE_CASE_: str  		=       BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp")
									SCREAMING_SNAKE_CASE_: List[Any]  		=       tokenizer.subword_tokenizer

									SCREAMING_SNAKE_CASE_: List[Any]  		=       subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。")
									self.assertListEqual(lowerCAmelCase__   ,						["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"])

									SCREAMING_SNAKE_CASE_: str  		=       subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは")
									self.assertListEqual(lowerCAmelCase__   ,						["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"])




			def        _SCREAMING_SNAKE_CASE			( self  :     Union[str, Any]):
									SCREAMING_SNAKE_CASE_: Dict  		=       self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese")

									SCREAMING_SNAKE_CASE_: Tuple  		=       tokenizer.encode("ありがとう。"   ,						add_special_tokens=lowerCAmelCase__)
									SCREAMING_SNAKE_CASE_: Dict  		=       tokenizer.encode("どういたしまして。"   ,						add_special_tokens=lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_: Dict  		=       tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__)
									SCREAMING_SNAKE_CASE_: int  		=       tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__   ,						lowerCAmelCase__)

									# 2 is for "[CLS]", 3 is for "[SEP]"
									assert encoded_sentence == [2] + text + [3]
									assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class   __lowercase				( UpperCAmelCase_		,       unittest.TestCase	):




			"""simple docstring"""







			_UpperCAmelCase   :							Optional[Any]										=     BertJapaneseTokenizer
			_UpperCAmelCase   :							List[str]										=     False



			def        _SCREAMING_SNAKE_CASE			( self  :     List[Any]):
									super().setUp()

									SCREAMING_SNAKE_CASE_: Any  		=       ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

									SCREAMING_SNAKE_CASE_: int  		=       os.path.join(self.tmpdirname   ,						VOCAB_FILES_NAMES["vocab_file"])
									with open(self.vocab_file   ,						"w"   ,						encoding="utf-8") as vocab_writer:
															vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))



			def        _SCREAMING_SNAKE_CASE			( self  :     List[str]   ,						**lowerCAmelCase__  :     Union[str, Any]):
									return BertJapaneseTokenizer.from_pretrained(self.tmpdirname   ,						subword_tokenizer_type="character"   ,						**lowerCAmelCase__)



			def        _SCREAMING_SNAKE_CASE			( self  :     Optional[int]   ,						lowerCAmelCase__  :     Union[str, Any]):
									SCREAMING_SNAKE_CASE_: List[Any]  		=       "こんにちは、世界。 \nこんばんは、世界。"
									SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       "こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
									return input_text, output_text



			def        _SCREAMING_SNAKE_CASE			( self  :     Tuple):
									pass  # TODO add if relevant



			def        _SCREAMING_SNAKE_CASE			( self  :     int):
									pass  # TODO add if relevant



			def        _SCREAMING_SNAKE_CASE			( self  :     Any):
									pass  # TODO add if relevant



			def        _SCREAMING_SNAKE_CASE			( self  :     Optional[int]):
									SCREAMING_SNAKE_CASE_: Dict  		=       self.tokenizer_class(self.vocab_file   ,						subword_tokenizer_type="character")

									SCREAMING_SNAKE_CASE_: Optional[int]  		=       tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。")
									self.assertListEqual(
									    lowerCAmelCase__   ,						["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"])
									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(lowerCAmelCase__)   ,						[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12])



			def        _SCREAMING_SNAKE_CASE			( self  :     Optional[Any]):
									SCREAMING_SNAKE_CASE_: Optional[Any]  		=       ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

									SCREAMING_SNAKE_CASE_: List[Any]  		=       {}
									for i, token in enumerate(lowerCAmelCase__):
															SCREAMING_SNAKE_CASE_: Dict  		=       i
									SCREAMING_SNAKE_CASE_: Optional[int]  		=       CharacterTokenizer(vocab=lowerCAmelCase__   ,						unk_token="[UNK]")

									self.assertListEqual(tokenizer.tokenize("")   ,						[])

									self.assertListEqual(tokenizer.tokenize("こんにちは")   ,						["こ", "ん", "に", "ち", "は"])

									self.assertListEqual(tokenizer.tokenize("こんにちほ")   ,						["こ", "ん", "に", "ち", "[UNK]"])




			def        _SCREAMING_SNAKE_CASE			( self  :     Optional[Any]):
									SCREAMING_SNAKE_CASE_: str  		=       self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char")

									SCREAMING_SNAKE_CASE_: Optional[int]  		=       tokenizer.encode("ありがとう。"   ,						add_special_tokens=lowerCAmelCase__)
									SCREAMING_SNAKE_CASE_: List[Any]  		=       tokenizer.encode("どういたしまして。"   ,						add_special_tokens=lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_: int  		=       tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__)
									SCREAMING_SNAKE_CASE_: List[Any]  		=       tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__   ,						lowerCAmelCase__)

									# 2 is for "[CLS]", 3 is for "[SEP]"
									assert encoded_sentence == [2] + text + [3]
									assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class   __lowercase				( unittest.TestCase	):




			"""simple docstring"""







			def        _SCREAMING_SNAKE_CASE			( self  :     str):
									SCREAMING_SNAKE_CASE_: List[str]  		=       "cl-tohoku/bert-base-japanese"
									SCREAMING_SNAKE_CASE_: List[str]  		=       AutoTokenizer.from_pretrained(lowerCAmelCase__)
									self.assertIsInstance(lowerCAmelCase__   ,						lowerCAmelCase__)







class   __lowercase				( unittest.TestCase	):




			"""simple docstring"""







			def        _SCREAMING_SNAKE_CASE			( self  :     Union[str, Any]):
									SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       "cl-tohoku/bert-base-japanese"
									with self.assertLogs("transformers"   ,						level="WARNING") as cm:
															BertTokenizer.from_pretrained(lowerCAmelCase__)
															self.assertTrue(
															    cm.records[0].message.startswith(
															        "The tokenizer class you load from this checkpoint is not the same type as the class this function"
															        " is called from."))
									SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       "bert-base-cased"
									with self.assertLogs("transformers"   ,						level="WARNING") as cm:
															BertJapaneseTokenizer.from_pretrained(lowerCAmelCase__)
															self.assertTrue(
															    cm.records[0].message.startswith(
															        "The tokenizer class you load from this checkpoint is not the same type as the class this function"
															        " is called from."))









class   __lowercase				:




			"""simple docstring"""







			def __init__( self  :     List[Any]   ,						lowerCAmelCase__  :     Optional[int]   ,						lowerCAmelCase__  :     List[Any]):
									SCREAMING_SNAKE_CASE_: List[str]  		=       name
									SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       val



			def __str__( self  :     Dict):
									return F"{self.__class__.__name__}({self.name}, {self.val})"




			def __lt__( self  :     List[str]   ,						lowerCAmelCase__  :     Any):
									return self.val < other.val







class   __lowercase				:




			"""simple docstring"""







			def __init__( self  :     Tuple   ,						lowerCAmelCase__  :     Dict):
									SCREAMING_SNAKE_CASE_: str  		=       {}
									SCREAMING_SNAKE_CASE_: int  		=       {}
									SCREAMING_SNAKE_CASE_: Any  		=       self.build_heap(lowerCAmelCase__)



			def __getitem__( self  :     List[Any]   ,						lowerCAmelCase__  :     Dict):
									return self.get_value(lowerCAmelCase__)



			def        _SCREAMING_SNAKE_CASE			( self  :     Optional[Any]   ,						lowerCAmelCase__  :     Dict):
									return (idx - 1) // 2



			def        _SCREAMING_SNAKE_CASE			( self  :     Union[str, Any]   ,						lowerCAmelCase__  :     Optional[Any]):
									return idx * 2 + 1



			def        _SCREAMING_SNAKE_CASE			( self  :     Optional[int]   ,						lowerCAmelCase__  :     Tuple):
									return idx * 2 + 2



			def        _SCREAMING_SNAKE_CASE			( self  :     Optional[int]   ,						lowerCAmelCase__  :     Optional[int]):
									return self.heap_dict[key]



			def        _SCREAMING_SNAKE_CASE			( self  :     List[Any]   ,						lowerCAmelCase__  :     Union[str, Any]):
									SCREAMING_SNAKE_CASE_: Tuple  		=       len(lowerCAmelCase__) - 1
									SCREAMING_SNAKE_CASE_: List[str]  		=       self.get_parent_idx(lowerCAmelCase__)

									for idx, i in enumerate(lowerCAmelCase__):
															SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       idx
															SCREAMING_SNAKE_CASE_: str  		=       i.val

									for i in range(lowerCAmelCase__   ,						-1   ,						-1):
															self.sift_down(lowerCAmelCase__   ,						lowerCAmelCase__)
									return array



			def        _SCREAMING_SNAKE_CASE			( self  :     str   ,						lowerCAmelCase__  :     Optional[int]   ,						lowerCAmelCase__  :     List[str]):
									while True:
															SCREAMING_SNAKE_CASE_: Optional[Any]  		=       self.get_left_child_idx(lowerCAmelCase__)  # noqa: E741
															SCREAMING_SNAKE_CASE_: Dict  		=       self.get_right_child_idx(lowerCAmelCase__)

															SCREAMING_SNAKE_CASE_: int  		=       idx
															if l < len(lowerCAmelCase__) and array[l] < array[idx]:
																					SCREAMING_SNAKE_CASE_: List[str]  		=       l
															if r < len(lowerCAmelCase__) and array[r] < array[smallest]:
																					SCREAMING_SNAKE_CASE_: str  		=       r

															if smallest != idx:
																					SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Any  		=       array[smallest], array[idx]
																					(
																					    (
																					    SCREAMING_SNAKE_CASE_
																					)		,    (
																					    SCREAMING_SNAKE_CASE_
																					)		,    
																					): Optional[Any]  		=       (
																					    self.idx_of_element[array[smallest]],
																					    self.idx_of_element[array[idx]],
																					)
																					SCREAMING_SNAKE_CASE_: Optional[int]  		=       smallest
															else:
																					break



			def        _SCREAMING_SNAKE_CASE			( self  :     Dict   ,						lowerCAmelCase__  :     str):
									SCREAMING_SNAKE_CASE_: Any  		=       self.get_parent_idx(lowerCAmelCase__)
									while p >= 0 and self.heap[p] > self.heap[idx]:
															SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: List[Any]  		=       self.heap[idx], self.heap[p]
															SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Tuple  		=       (
															    self.idx_of_element[self.heap[idx]],
															    self.idx_of_element[self.heap[p]],
															)
															SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       p
															SCREAMING_SNAKE_CASE_: Optional[int]  		=       self.get_parent_idx(lowerCAmelCase__)



			def        _SCREAMING_SNAKE_CASE			( self  :     List[Any]):
									return self.heap[0]



			def        _SCREAMING_SNAKE_CASE			( self  :     Dict):
									SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Tuple  		=       self.heap[-1], self.heap[0]
									SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: List[str]  		=       (
									    self.idx_of_element[self.heap[-1]],
									    self.idx_of_element[self.heap[0]],
									)

									SCREAMING_SNAKE_CASE_: int  		=       self.heap.pop()
									del self.idx_of_element[x]
									self.sift_down(0   ,						self.heap)
									return x



			def        _SCREAMING_SNAKE_CASE			( self  :     Union[str, Any]   ,						lowerCAmelCase__  :     Tuple):
									self.heap.append(lowerCAmelCase__)
									SCREAMING_SNAKE_CASE_: List[str]  		=       len(self.heap) - 1
									SCREAMING_SNAKE_CASE_: List[str]  		=       node.val
									self.sift_up(len(self.heap) - 1)



			def        _SCREAMING_SNAKE_CASE			( self  :     List[Any]):
									return len(self.heap) == 0




			def        _SCREAMING_SNAKE_CASE			( self  :     Optional[Any]   ,						lowerCAmelCase__  :     Any   ,						lowerCAmelCase__  :     Optional[int]):
									assert (
									    self.heap[self.idx_of_element[node]].val > new_value
									), "newValue must be less that current value"
									SCREAMING_SNAKE_CASE_: Any  		=       new_value
									SCREAMING_SNAKE_CASE_: Tuple  		=       new_value
									self.sift_up(self.idx_of_element[node])







lowerCAmelCase							:  int   	=						Node("""R""", -1)
lowerCAmelCase							:  str   	=						Node("""B""", 6)
lowerCAmelCase							:  str   	=						Node("""A""", 3)
lowerCAmelCase							:  List[str]   	=						Node("""X""", 1)
lowerCAmelCase							:  Union[str, Any]   	=						Node("""E""", 4)

# Use one of these two ways to generate Min-Heap

# Generating Min-Heap from array
lowerCAmelCase							:  Optional[Any]   	=						MinHeap([r, b, a, x, e])

# Generating Min-Heap by Insert method
# myMinHeap.insert(a)
# myMinHeap.insert(b)
# myMinHeap.insert(x)
# myMinHeap.insert(r)
# myMinHeap.insert(e)

# Before
print("""Min Heap - before decrease key""")
for i in my_min_heap.heap:
						print(i)

print("""Min Heap - After decrease key of node [B -> -17]""")
my_min_heap.decrease_key(b, -17)

# After
for i in my_min_heap.heap:
						print(i)

if __name__ == "__main__":
						import doctest

						doctest.testmod()






