


"""simple docstring"""




import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin
@custom_tokenizers
class 					lowerCAmelCase	(					_UpperCAmelCase	,    unittest.TestCase    ):




		'''simple docstring'''







		SCREAMING_SNAKE_CASE_						:     Optional[int]								=     BertJapaneseTokenizer
		SCREAMING_SNAKE_CASE_						:     Tuple								=     False
		SCREAMING_SNAKE_CASE_						:     List[Any]								=     True







		def 							__A						(	self				)     ->							Optional[int]:
								super().setUp()

								SCREAMING_SNAKE_CASE													=					[
								    '[UNK]',
								    '[CLS]',
								    '[SEP]',
								    'こんにちは',
								    'こん',
								    'にちは',
								    'ばんは',
								    '##こん',
								    '##にちは',
								    '##ばんは',
								    '世界',
								    '##世界',
								    '、',
								    '##、',
								    '。',
								    '##。',
								]

								SCREAMING_SNAKE_CASE													=					os.path.join(self.tmpdirname						,       VOCAB_FILES_NAMES['vocab_file']				)
								with open(self.vocab_file						,       'w'						,       encoding='utf-8'				) as vocab_writer:
														vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]				)				)







		def 							__A						(	self						,       lowerCAmelCase__				)     ->							Optional[int]:
								SCREAMING_SNAKE_CASE													=					'こんにちは、世界。 \nこんばんは、世界。'
								SCREAMING_SNAKE_CASE													=					'こんにちは 、 世界 。 こんばんは 、 世界 。'
								return input_text, output_text







		def 							__A						(	self						,       lowerCAmelCase__				)     ->							int:
								SCREAMING_SNAKE_CASE							,				SCREAMING_SNAKE_CASE													=					self.get_input_output_texts(lowerCAmelCase__				)
								SCREAMING_SNAKE_CASE													=					tokenizer.encode(lowerCAmelCase__						,       add_special_tokens=lowerCAmelCase__				)
								SCREAMING_SNAKE_CASE													=					tokenizer.decode(lowerCAmelCase__						,       clean_up_tokenization_spaces=lowerCAmelCase__				)
								return text, ids







		def 							__A						(	self				)     ->							Tuple:
								pass  # TODO add if relevant







		def 							__A						(	self				)     ->							Any:
								pass  # TODO add if relevant







		def 							__A						(	self				)     ->							int:
								pass  # TODO add if relevant







		def 							__A						(	self				)     ->							Any:
								SCREAMING_SNAKE_CASE													=					self.tokenizer_class(self.vocab_file				)

								SCREAMING_SNAKE_CASE													=					tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'				)
								self.assertListEqual(lowerCAmelCase__						,       ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']				)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__				)						,       [3, 12, 10, 14, 4, 9, 12, 10, 14]				)







		def 							__A						(	self				)     ->							List[Any]:
								SCREAMING_SNAKE_CASE													=					self.tokenizer_class(self.vocab_file						,       word_tokenizer_type='mecab'				)
								self.assertIsNotNone(lowerCAmelCase__				)

								SCREAMING_SNAKE_CASE													=					'こんにちは、世界。\nこんばんは、世界。'
								SCREAMING_SNAKE_CASE													=					tokenizer.tokenize(lowerCAmelCase__				)
								self.assertListEqual(lowerCAmelCase__						,       ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']				)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__				)						,       [3, 12, 10, 14, 4, 9, 12, 10, 14]				)

								SCREAMING_SNAKE_CASE													=					os.path.join(self.tmpdirname						,       'tokenizer.bin'				)
								with open(lowerCAmelCase__						,       'wb'				) as handle:
														pickle.dump(lowerCAmelCase__						,       lowerCAmelCase__				)

								with open(lowerCAmelCase__						,       'rb'				) as handle:
														SCREAMING_SNAKE_CASE													=					pickle.load(lowerCAmelCase__				)

								SCREAMING_SNAKE_CASE													=					tokenizer_new.tokenize(lowerCAmelCase__				)

								self.assertListEqual(lowerCAmelCase__						,       lowerCAmelCase__				)







		def 							__A						(	self				)     ->							List[str]:
								SCREAMING_SNAKE_CASE													=					MecabTokenizer(mecab_dic='ipadic'				)

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '				)						,       ['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']						,       )







		def 							__A						(	self				)     ->							List[Any]:
								try:
														SCREAMING_SNAKE_CASE													=					MecabTokenizer(mecab_dic='unidic_lite'				)
								except ModuleNotFoundError:
														return

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '				)						,       ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']						,       )







		def 							__A						(	self				)     ->							Any:
								try:
														SCREAMING_SNAKE_CASE													=					MecabTokenizer(mecab_dic='unidic'				)
								except ModuleNotFoundError:
														return

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '				)						,       ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']						,       )







		def 							__A						(	self				)     ->							List[Any]:
								SCREAMING_SNAKE_CASE													=					MecabTokenizer(do_lower_case=lowerCAmelCase__						,       mecab_dic='ipadic'				)

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '				)						,       ['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']						,       )







		def 							__A						(	self				)     ->							Union[str, Any]:
								try:
														SCREAMING_SNAKE_CASE													=					MecabTokenizer(
														    do_lower_case=lowerCAmelCase__						,       normalize_text=lowerCAmelCase__						,       mecab_option='-d /usr/local/lib/mecab/dic/jumandic'				)
								except RuntimeError:
														# if dict doesn't exist in the system, previous code raises this error.
														return

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '				)						,       ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']						,       )







		def 							__A						(	self				)     ->							Dict:
								SCREAMING_SNAKE_CASE													=					MecabTokenizer(normalize_text=lowerCAmelCase__						,       mecab_dic='ipadic'				)

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '				)						,       ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']						,       )







		@require_sudachi
		def 							__A						(	self				)     ->							List[Any]:
								SCREAMING_SNAKE_CASE													=					self.tokenizer_class(self.vocab_file						,       word_tokenizer_type='sudachi'				)
								self.assertIsNotNone(lowerCAmelCase__				)

								SCREAMING_SNAKE_CASE													=					'こんにちは、世界。\nこんばんは、世界。'
								SCREAMING_SNAKE_CASE													=					tokenizer.tokenize(lowerCAmelCase__				)
								self.assertListEqual(lowerCAmelCase__						,       ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']				)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__				)						,       [3, 12, 10, 14, 4, 9, 12, 10, 14]				)

								SCREAMING_SNAKE_CASE													=					os.path.join(self.tmpdirname						,       'tokenizer.bin'				)
								with open(lowerCAmelCase__						,       'wb'				) as handle:
														pickle.dump(lowerCAmelCase__						,       lowerCAmelCase__				)

								with open(lowerCAmelCase__						,       'rb'				) as handle:
														SCREAMING_SNAKE_CASE													=					pickle.load(lowerCAmelCase__				)

								SCREAMING_SNAKE_CASE													=					tokenizer_new.tokenize(lowerCAmelCase__				)

								self.assertListEqual(lowerCAmelCase__						,       lowerCAmelCase__				)







		@require_sudachi
		def 							__A						(	self				)     ->							List[str]:
								SCREAMING_SNAKE_CASE													=					SudachiTokenizer(sudachi_dict_type='core'				)

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '				)						,       [' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']						,       )







		@require_sudachi
		def 							__A						(	self				)     ->							Any:
								SCREAMING_SNAKE_CASE													=					SudachiTokenizer(sudachi_dict_type='core'						,       sudachi_split_mode='A'				)

								self.assertListEqual(tokenizer.tokenize('外国人参政権'				)						,       ['外国', '人', '参政', '権']				)







		@require_sudachi
		def 							__A						(	self				)     ->							Optional[Any]:
								SCREAMING_SNAKE_CASE													=					SudachiTokenizer(sudachi_dict_type='core'						,       sudachi_split_mode='B'				)

								self.assertListEqual(tokenizer.tokenize('外国人参政権'				)						,       ['外国人', '参政権']				)







		@require_sudachi
		def 							__A						(	self				)     ->							Optional[Any]:
								SCREAMING_SNAKE_CASE													=					SudachiTokenizer(sudachi_dict_type='core'						,       sudachi_split_mode='C'				)

								self.assertListEqual(tokenizer.tokenize('外国人参政権'				)						,       ['外国人参政権']				)







		@require_sudachi
		def 							__A						(	self				)     ->							Optional[int]:
								SCREAMING_SNAKE_CASE													=					SudachiTokenizer(do_lower_case=lowerCAmelCase__						,       sudachi_dict_type='core'				)

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '				)						,       [' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']						,       )







		@require_sudachi
		def 							__A						(	self				)     ->							Union[str, Any]:
								SCREAMING_SNAKE_CASE													=					SudachiTokenizer(normalize_text=lowerCAmelCase__						,       sudachi_dict_type='core'				)

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '				)						,       [' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']						,       )







		@require_sudachi
		def 							__A						(	self				)     ->							List[str]:
								SCREAMING_SNAKE_CASE													=					SudachiTokenizer(trim_whitespace=lowerCAmelCase__						,       sudachi_dict_type='core'				)

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '				)						,       ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']						,       )







		@require_jumanpp
		def 							__A						(	self				)     ->							int:
								SCREAMING_SNAKE_CASE													=					self.tokenizer_class(self.vocab_file						,       word_tokenizer_type='jumanpp'				)
								self.assertIsNotNone(lowerCAmelCase__				)

								SCREAMING_SNAKE_CASE													=					'こんにちは、世界。\nこんばんは、世界。'
								SCREAMING_SNAKE_CASE													=					tokenizer.tokenize(lowerCAmelCase__				)
								self.assertListEqual(lowerCAmelCase__						,       ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']				)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__				)						,       [3, 12, 10, 14, 4, 9, 12, 10, 14]				)

								SCREAMING_SNAKE_CASE													=					os.path.join(self.tmpdirname						,       'tokenizer.bin'				)
								with open(lowerCAmelCase__						,       'wb'				) as handle:
														pickle.dump(lowerCAmelCase__						,       lowerCAmelCase__				)

								with open(lowerCAmelCase__						,       'rb'				) as handle:
														SCREAMING_SNAKE_CASE													=					pickle.load(lowerCAmelCase__				)

								SCREAMING_SNAKE_CASE													=					tokenizer_new.tokenize(lowerCAmelCase__				)

								self.assertListEqual(lowerCAmelCase__						,       lowerCAmelCase__				)







		@require_jumanpp
		def 							__A						(	self				)     ->							Dict:
								SCREAMING_SNAKE_CASE													=					JumanppTokenizer()

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '				)						,       ['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']						,       )







		@require_jumanpp
		def 							__A						(	self				)     ->							Optional[int]:
								SCREAMING_SNAKE_CASE													=					JumanppTokenizer(do_lower_case=lowerCAmelCase__				)

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '				)						,       ['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']						,       )







		@require_jumanpp
		def 							__A						(	self				)     ->							Dict:
								SCREAMING_SNAKE_CASE													=					JumanppTokenizer(normalize_text=lowerCAmelCase__				)

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '				)						,       ['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']						,       )







		@require_jumanpp
		def 							__A						(	self				)     ->							Tuple:
								SCREAMING_SNAKE_CASE													=					JumanppTokenizer(trim_whitespace=lowerCAmelCase__				)

								self.assertListEqual(
								    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '				)						,       ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']						,       )







		@require_jumanpp
		def 							__A						(	self				)     ->							Dict:
								SCREAMING_SNAKE_CASE													=					JumanppTokenizer()

								self.assertListEqual(
								    tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'				)						,       ['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']						,       )







		def 							__A						(	self				)     ->							Tuple:
								SCREAMING_SNAKE_CASE													=					['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

								SCREAMING_SNAKE_CASE													=					{}
								for i, token in enumerate(lowerCAmelCase__				):
														SCREAMING_SNAKE_CASE													=					i
								SCREAMING_SNAKE_CASE													=					WordpieceTokenizer(vocab=lowerCAmelCase__						,       unk_token='[UNK]'				)

								self.assertListEqual(tokenizer.tokenize(''				)						,       []				)

								self.assertListEqual(tokenizer.tokenize('こんにちは'				)						,       ['こんにちは']				)

								self.assertListEqual(tokenizer.tokenize('こんばんは'				)						,       ['こん', '##ばんは']				)

								self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'				)						,       ['こん', '##ばんは', '[UNK]', 'こんにちは']				)







		def 							__A						(	self				)     ->							List[str]:
								SCREAMING_SNAKE_CASE													=					BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'				)
								SCREAMING_SNAKE_CASE													=					tokenizer.subword_tokenizer

								SCREAMING_SNAKE_CASE													=					subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'				)
								self.assertListEqual(lowerCAmelCase__						,       ['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']				)

								SCREAMING_SNAKE_CASE													=					subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'				)
								self.assertListEqual(lowerCAmelCase__						,       ['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']				)







		def 							__A						(	self				)     ->							Tuple:
								SCREAMING_SNAKE_CASE													=					self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'				)

								SCREAMING_SNAKE_CASE													=					tokenizer.encode('ありがとう。'						,       add_special_tokens=lowerCAmelCase__				)
								SCREAMING_SNAKE_CASE													=					tokenizer.encode('どういたしまして。'						,       add_special_tokens=lowerCAmelCase__				)

								SCREAMING_SNAKE_CASE													=					tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__				)
								SCREAMING_SNAKE_CASE													=					tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__						,       lowerCAmelCase__				)

								# 2 is for "[CLS]", 3 is for "[SEP]"
								assert encoded_sentence == [2] + text + [3]
								assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 					lowerCAmelCase	(					_UpperCAmelCase	,    unittest.TestCase    ):




		'''simple docstring'''







		SCREAMING_SNAKE_CASE_						:     Union[str, Any]								=     BertJapaneseTokenizer
		SCREAMING_SNAKE_CASE_						:     Dict								=     False







		def 							__A						(	self				)     ->							Optional[int]:
								super().setUp()

								SCREAMING_SNAKE_CASE													=					['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

								SCREAMING_SNAKE_CASE													=					os.path.join(self.tmpdirname						,       VOCAB_FILES_NAMES['vocab_file']				)
								with open(self.vocab_file						,       'w'						,       encoding='utf-8'				) as vocab_writer:
														vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]				)				)







		def 							__A						(	self						,       **lowerCAmelCase__				)     ->							Optional[int]:
								return BertJapaneseTokenizer.from_pretrained(self.tmpdirname						,       subword_tokenizer_type='character'						,       **lowerCAmelCase__				)







		def 							__A						(	self						,       lowerCAmelCase__				)     ->							Union[str, Any]:
								SCREAMING_SNAKE_CASE													=					'こんにちは、世界。 \nこんばんは、世界。'
								SCREAMING_SNAKE_CASE													=					'こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
								return input_text, output_text







		def 							__A						(	self				)     ->							Dict:
								pass  # TODO add if relevant







		def 							__A						(	self				)     ->							Tuple:
								pass  # TODO add if relevant







		def 							__A						(	self				)     ->							Tuple:
								pass  # TODO add if relevant







		def 							__A						(	self				)     ->							Tuple:
								SCREAMING_SNAKE_CASE													=					self.tokenizer_class(self.vocab_file						,       subword_tokenizer_type='character'				)

								SCREAMING_SNAKE_CASE													=					tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'				)
								self.assertListEqual(
								    lowerCAmelCase__						,       ['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']				)
								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(lowerCAmelCase__				)						,       [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]				)







		def 							__A						(	self				)     ->							Optional[int]:
								SCREAMING_SNAKE_CASE													=					['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

								SCREAMING_SNAKE_CASE													=					{}
								for i, token in enumerate(lowerCAmelCase__				):
														SCREAMING_SNAKE_CASE													=					i
								SCREAMING_SNAKE_CASE													=					CharacterTokenizer(vocab=lowerCAmelCase__						,       unk_token='[UNK]'				)

								self.assertListEqual(tokenizer.tokenize(''				)						,       []				)

								self.assertListEqual(tokenizer.tokenize('こんにちは'				)						,       ['こ', 'ん', 'に', 'ち', 'は']				)

								self.assertListEqual(tokenizer.tokenize('こんにちほ'				)						,       ['こ', 'ん', 'に', 'ち', '[UNK]']				)







		def 							__A						(	self				)     ->							Dict:
								SCREAMING_SNAKE_CASE													=					self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'				)

								SCREAMING_SNAKE_CASE													=					tokenizer.encode('ありがとう。'						,       add_special_tokens=lowerCAmelCase__				)
								SCREAMING_SNAKE_CASE													=					tokenizer.encode('どういたしまして。'						,       add_special_tokens=lowerCAmelCase__				)

								SCREAMING_SNAKE_CASE													=					tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__				)
								SCREAMING_SNAKE_CASE													=					tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__						,       lowerCAmelCase__				)

								# 2 is for "[CLS]", 3 is for "[SEP]"
								assert encoded_sentence == [2] + text + [3]
								assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 					lowerCAmelCase	(					unittest.TestCase    ):




		'''simple docstring'''







		def 							__A						(	self				)     ->							Union[str, Any]:
								SCREAMING_SNAKE_CASE													=					'cl-tohoku/bert-base-japanese'
								SCREAMING_SNAKE_CASE													=					AutoTokenizer.from_pretrained(lowerCAmelCase__				)
								self.assertIsInstance(lowerCAmelCase__						,       lowerCAmelCase__				)





class 					lowerCAmelCase	(					unittest.TestCase    ):




		'''simple docstring'''







		def 							__A						(	self				)     ->							Dict:
								SCREAMING_SNAKE_CASE													=					'cl-tohoku/bert-base-japanese'
								with self.assertLogs('transformers'						,       level='WARNING'				) as cm:
														BertTokenizer.from_pretrained(lowerCAmelCase__				)
														self.assertTrue(
														    cm.records[0].message.startswith(
														        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
														        ' is called from.'				)				)
								SCREAMING_SNAKE_CASE													=					'bert-base-cased'
								with self.assertLogs('transformers'						,       level='WARNING'				) as cm:
														BertJapaneseTokenizer.from_pretrained(lowerCAmelCase__				)
														self.assertTrue(
														    cm.records[0].message.startswith(
														        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
														        ' is called from.'				)				)









"""simple docstring"""





import tempfile

import numpy as np
import torch
from transformers import AutoTokenizer, TaEncoderModel

from diffusers import DDPMScheduler, UNetaDConditionModel
from diffusers.models.attention_processor import AttnAddedKVProcessor
from diffusers.pipelines.deepfloyd_if import IFWatermarker
from diffusers.utils.testing_utils import torch_device

from ..test_pipelines_common import to_np

class 		lowercase					:





      def     _snake_case			(       self )		->     List[Any]:
            torch.manual_seed(0 )
            lowerCAmelCase  						=       TaEncoderModel.from_pretrained("""hf-internal-testing/tiny-random-t5""" )

            torch.manual_seed(0 )
            lowerCAmelCase  						=       AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-t5""" )

            torch.manual_seed(0 )
            lowerCAmelCase  						=       UNetaDConditionModel(
                sample_size=32      ,						layers_per_block=1      ,						block_out_channels=[32, 64]      ,						down_block_types=[
                    """ResnetDownsampleBlock2D""",
                    """SimpleCrossAttnDownBlock2D""",
                ]      ,						mid_block_type="""UNetMidBlock2DSimpleCrossAttn"""      ,						up_block_types=["""SimpleCrossAttnUpBlock2D""", """ResnetUpsampleBlock2D"""]      ,						in_channels=3      ,						out_channels=6      ,						cross_attention_dim=32      ,						encoder_hid_dim=32      ,						attention_head_dim=8      ,						addition_embed_type="""text"""      ,						addition_embed_type_num_heads=2      ,						cross_attention_norm="""group_norm"""      ,						resnet_time_scale_shift="""scale_shift"""      ,						act_fn="""gelu"""      ,						)
            unet.set_attn_processor(AttnAddedKVProcessor() )  # For reproducibility tests

            torch.manual_seed(0 )
            lowerCAmelCase  						=       DDPMScheduler(
                num_train_timesteps=1_000      ,						beta_schedule="""squaredcos_cap_v2"""      ,						beta_start=0.0_001      ,						beta_end=0.02      ,						thresholding=lowercase      ,						dynamic_thresholding_ratio=0.95      ,						sample_max_value=1.0      ,						prediction_type="""epsilon"""      ,						variance_type="""learned_range"""      ,						)

            torch.manual_seed(0 )
            lowerCAmelCase  						=       IFWatermarker()

            return {
                "text_encoder": text_encoder,
                "tokenizer": tokenizer,
                "unet": unet,
                "scheduler": scheduler,
                "watermarker": watermarker,
                "safety_checker": None,
                "feature_extractor": None,
            }





      def     _snake_case			(       self )		->     int:
            torch.manual_seed(0 )
            lowerCAmelCase  						=       TaEncoderModel.from_pretrained("""hf-internal-testing/tiny-random-t5""" )

            torch.manual_seed(0 )
            lowerCAmelCase  						=       AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-t5""" )

            torch.manual_seed(0 )
            lowerCAmelCase  						=       UNetaDConditionModel(
                sample_size=32      ,						layers_per_block=[1, 2]      ,						block_out_channels=[32, 64]      ,						down_block_types=[
                    """ResnetDownsampleBlock2D""",
                    """SimpleCrossAttnDownBlock2D""",
                ]      ,						mid_block_type="""UNetMidBlock2DSimpleCrossAttn"""      ,						up_block_types=["""SimpleCrossAttnUpBlock2D""", """ResnetUpsampleBlock2D"""]      ,						in_channels=6      ,						out_channels=6      ,						cross_attention_dim=32      ,						encoder_hid_dim=32      ,						attention_head_dim=8      ,						addition_embed_type="""text"""      ,						addition_embed_type_num_heads=2      ,						cross_attention_norm="""group_norm"""      ,						resnet_time_scale_shift="""scale_shift"""      ,						act_fn="""gelu"""      ,						class_embed_type="""timestep"""      ,						mid_block_scale_factor=1.414      ,						time_embedding_act_fn="""gelu"""      ,						time_embedding_dim=32      ,						)
            unet.set_attn_processor(AttnAddedKVProcessor() )  # For reproducibility tests

            torch.manual_seed(0 )
            lowerCAmelCase  						=       DDPMScheduler(
                num_train_timesteps=1_000      ,						beta_schedule="""squaredcos_cap_v2"""      ,						beta_start=0.0_001      ,						beta_end=0.02      ,						thresholding=lowercase      ,						dynamic_thresholding_ratio=0.95      ,						sample_max_value=1.0      ,						prediction_type="""epsilon"""      ,						variance_type="""learned_range"""      ,						)

            torch.manual_seed(0 )
            lowerCAmelCase  						=       DDPMScheduler(
                num_train_timesteps=1_000      ,						beta_schedule="""squaredcos_cap_v2"""      ,						beta_start=0.0_001      ,						beta_end=0.02      ,						)

            torch.manual_seed(0 )
            lowerCAmelCase  						=       IFWatermarker()

            return {
                "text_encoder": text_encoder,
                "tokenizer": tokenizer,
                "unet": unet,
                "scheduler": scheduler,
                "image_noising_scheduler": image_noising_scheduler,
                "watermarker": watermarker,
                "safety_checker": None,
                "feature_extractor": None,
            }





      def     _snake_case			(       self )		->     Union[str, Any]:
            lowerCAmelCase  						=       self.get_dummy_components()
            lowerCAmelCase  						=       self.pipeline_class(**lowercase )
            pipe.to(lowercase )
            pipe.set_progress_bar_config(disable=lowercase )

            lowerCAmelCase  						=       self.get_dummy_inputs(lowercase )

            lowerCAmelCase  						=       inputs["""prompt"""]
            lowerCAmelCase  						=       inputs["""generator"""]
            lowerCAmelCase  						=       inputs["""num_inference_steps"""]
            lowerCAmelCase  						=       inputs["""output_type"""]

            if "image" in inputs:
                  lowerCAmelCase  						=       inputs["""image"""]
            else:
                  lowerCAmelCase  						=       None

            if "mask_image" in inputs:
                  lowerCAmelCase  						=       inputs["""mask_image"""]
            else:
                  lowerCAmelCase  						=       None

            if "original_image" in inputs:
                  lowerCAmelCase  						=       inputs["""original_image"""]
            else:
                  lowerCAmelCase  						=       None

            lowerCAmelCase						,		lowerCAmelCase  						=       pipe.encode_prompt(lowercase )

            # inputs with prompt converted to embeddings
            lowerCAmelCase  						=       {
                """prompt_embeds""": prompt_embeds,
                """negative_prompt_embeds""": negative_prompt_embeds,
                """generator""": generator,
                """num_inference_steps""": num_inference_steps,
                """output_type""": output_type,
            }

            if image is not None:
                  lowerCAmelCase  						=       image

            if mask_image is not None:
                  lowerCAmelCase  						=       mask_image

            if original_image is not None:
                  lowerCAmelCase  						=       original_image

            # set all optional components to None
            for optional_component in pipe._optional_components:
                  setattr(lowercase      ,						lowercase      ,						lowercase )

            lowerCAmelCase  						=       pipe(**lowercase )[0]

            with tempfile.TemporaryDirectory() as tmpdir:
                  pipe.save_pretrained(lowercase )
                  lowerCAmelCase  						=       self.pipeline_class.from_pretrained(lowercase )
                  pipe_loaded.to(lowercase )
                  pipe_loaded.set_progress_bar_config(disable=lowercase )

            pipe_loaded.unet.set_attn_processor(AttnAddedKVProcessor() )  # For reproducibility tests

            for optional_component in pipe._optional_components:
                  self.assertTrue(
                      getattr(lowercase      ,						lowercase ) is None      ,						f'`{optional_component}` did not stay set to None after loading.'      ,						)

            lowerCAmelCase  						=       self.get_dummy_inputs(lowercase )

            lowerCAmelCase  						=       inputs["""generator"""]
            lowerCAmelCase  						=       inputs["""num_inference_steps"""]
            lowerCAmelCase  						=       inputs["""output_type"""]

            # inputs with prompt converted to embeddings
            lowerCAmelCase  						=       {
                """prompt_embeds""": prompt_embeds,
                """negative_prompt_embeds""": negative_prompt_embeds,
                """generator""": generator,
                """num_inference_steps""": num_inference_steps,
                """output_type""": output_type,
            }

            if image is not None:
                  lowerCAmelCase  						=       image

            if mask_image is not None:
                  lowerCAmelCase  						=       mask_image

            if original_image is not None:
                  lowerCAmelCase  						=       original_image

            lowerCAmelCase  						=       pipe_loaded(**lowercase )[0]

            lowerCAmelCase  						=       np.abs(to_np(lowercase ) - to_np(lowercase ) ).max()
            self.assertLess(lowercase      ,						1e-4 )





      def     _snake_case			(       self )		->     Optional[int]:
            lowerCAmelCase  						=       self.get_dummy_components()
            lowerCAmelCase  						=       self.pipeline_class(**lowercase )
            pipe.to(lowercase )
            pipe.set_progress_bar_config(disable=lowercase )

            lowerCAmelCase  						=       self.get_dummy_inputs(lowercase )
            lowerCAmelCase  						=       pipe(**lowercase )[0]

            with tempfile.TemporaryDirectory() as tmpdir:
                  pipe.save_pretrained(lowercase )
                  lowerCAmelCase  						=       self.pipeline_class.from_pretrained(lowercase )
                  pipe_loaded.to(lowercase )
                  pipe_loaded.set_progress_bar_config(disable=lowercase )

            pipe_loaded.unet.set_attn_processor(AttnAddedKVProcessor() )  # For reproducibility tests

            lowerCAmelCase  						=       self.get_dummy_inputs(lowercase )
            lowerCAmelCase  						=       pipe_loaded(**lowercase )[0]

            lowerCAmelCase  						=       np.abs(to_np(lowercase ) - to_np(lowercase ) ).max()
            self.assertLess(lowercase      ,						1e-4 )






