


import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin



@custom_tokenizers
class 							A    (       UpperCAmelCase__ ,  unittest.TestCase							):
					'''simple docstring'''






					A__			= BertJapaneseTokenizer
					A__			= False
					A__			= True

					def   lowerCamelCase__     (self    :						List[str]		)						->   Tuple:




							"""simple docstring"""


							super().setUp()

							lowercase__     			=		[
							    """[UNK]""",
							    """[CLS]""",
							    """[SEP]""",
							    """こんにちは""",
							    """こん""",
							    """にちは""",
							    """ばんは""",
							    """##こん""",
							    """##にちは""",
							    """##ばんは""",
							    """世界""",
							    """##世界""",
							    """、""",
							    """##、""",
							    """。""",
							    """##。""",
							]

							lowercase__     			=		os.path.join(self.tmpdirname      ,					VOCAB_FILES_NAMES["""vocab_file"""]		)
							with open(self.vocab_file      ,					"""w"""      ,					encoding="""utf-8"""		) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]		)		)

					def   lowerCamelCase__     (self    :						Tuple      ,					_UpperCAmelCase    :						Optional[int]		)						->   Union[str, Any]:




							"""simple docstring"""


							lowercase__     			=		"""こんにちは、世界。 \nこんばんは、世界。"""
							lowercase__     			=		"""こんにちは 、 世界 。 こんばんは 、 世界 。"""
							return input_text, output_text

					def   lowerCamelCase__     (self    :						str      ,					_UpperCAmelCase    :						List[str]		)						->   Union[str, Any]:




							"""simple docstring"""


							lowercase__			,	lowercase__     			=		self.get_input_output_texts(__UpperCAmelCase		)
							lowercase__     			=		tokenizer.encode(__UpperCAmelCase      ,					add_special_tokens=__UpperCAmelCase		)
							lowercase__     			=		tokenizer.decode(__UpperCAmelCase      ,					clean_up_tokenization_spaces=__UpperCAmelCase		)
							return text, ids

					def   lowerCamelCase__     (self    :						Optional[Any]		)						->   Any:




							"""simple docstring"""


							pass  # TODO add if relevant

					def   lowerCamelCase__     (self    :						int		)						->   int:




							"""simple docstring"""


							pass  # TODO add if relevant

					def   lowerCamelCase__     (self    :						Optional[Any]		)						->   Tuple:




							"""simple docstring"""


							pass  # TODO add if relevant

					def   lowerCamelCase__     (self    :						Dict		)						->   Any:




							"""simple docstring"""


							lowercase__     			=		self.tokenizer_class(self.vocab_file		)

							lowercase__     			=		tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""		)
							self.assertListEqual(__UpperCAmelCase      ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]		)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase		)      ,					[3, 12, 10, 14, 4, 9, 12, 10, 14]		)

					def   lowerCamelCase__     (self    :						List[str]		)						->   Dict:




							"""simple docstring"""


							lowercase__     			=		self.tokenizer_class(self.vocab_file      ,					word_tokenizer_type="""mecab"""		)
							self.assertIsNotNone(__UpperCAmelCase		)

							lowercase__     			=		"""こんにちは、世界。\nこんばんは、世界。"""
							lowercase__     			=		tokenizer.tokenize(__UpperCAmelCase		)
							self.assertListEqual(__UpperCAmelCase      ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]		)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase		)      ,					[3, 12, 10, 14, 4, 9, 12, 10, 14]		)

							lowercase__     			=		os.path.join(self.tmpdirname      ,					"""tokenizer.bin"""		)
							with open(__UpperCAmelCase      ,					"""wb"""		) as handle:
									pickle.dump(__UpperCAmelCase      ,					__UpperCAmelCase		)

							with open(__UpperCAmelCase      ,					"""rb"""		) as handle:
									lowercase__     			=		pickle.load(__UpperCAmelCase		)

							lowercase__     			=		tokenizer_new.tokenize(__UpperCAmelCase		)

							self.assertListEqual(__UpperCAmelCase      ,					__UpperCAmelCase		)

					def   lowerCamelCase__     (self    :						List[Any]		)						->   str:




							"""simple docstring"""


							lowercase__     			=		MecabTokenizer(mecab_dic="""ipadic"""		)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """		)      ,					["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,					)

					def   lowerCamelCase__     (self    :						Any		)						->   Union[str, Any]:




							"""simple docstring"""


							try:
									lowercase__     			=		MecabTokenizer(mecab_dic="""unidic_lite"""		)
							except ModuleNotFoundError:
									return

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """		)      ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,					)

					def   lowerCamelCase__     (self    :						int		)						->   Optional[int]:




							"""simple docstring"""


							try:
									lowercase__     			=		MecabTokenizer(mecab_dic="""unidic"""		)
							except ModuleNotFoundError:
									return

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """		)      ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,					)

					def   lowerCamelCase__     (self    :						Union[str, Any]		)						->   List[Any]:




							"""simple docstring"""


							lowercase__     			=		MecabTokenizer(do_lower_case=__UpperCAmelCase      ,					mecab_dic="""ipadic"""		)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """		)      ,					["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,					)

					def   lowerCamelCase__     (self    :						str		)						->   Tuple:




							"""simple docstring"""


							try:
									lowercase__     			=		MecabTokenizer(
									    do_lower_case=__UpperCAmelCase      ,					normalize_text=__UpperCAmelCase      ,					mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""		)
							except RuntimeError:
									# if dict doesn't exist in the system, previous code raises this error.
									return

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """		)      ,					["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]      ,					)

					def   lowerCamelCase__     (self    :						List[Any]		)						->   Dict:




							"""simple docstring"""


							lowercase__     			=		MecabTokenizer(normalize_text=__UpperCAmelCase      ,					mecab_dic="""ipadic"""		)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """		)      ,					["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]      ,					)

					@require_sudachi
					def   lowerCamelCase__     (self    :						Optional[Any]		)						->   Dict:




							"""simple docstring"""


							lowercase__     			=		self.tokenizer_class(self.vocab_file      ,					word_tokenizer_type="""sudachi"""		)
							self.assertIsNotNone(__UpperCAmelCase		)

							lowercase__     			=		"""こんにちは、世界。\nこんばんは、世界。"""
							lowercase__     			=		tokenizer.tokenize(__UpperCAmelCase		)
							self.assertListEqual(__UpperCAmelCase      ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]		)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase		)      ,					[3, 12, 10, 14, 4, 9, 12, 10, 14]		)

							lowercase__     			=		os.path.join(self.tmpdirname      ,					"""tokenizer.bin"""		)
							with open(__UpperCAmelCase      ,					"""wb"""		) as handle:
									pickle.dump(__UpperCAmelCase      ,					__UpperCAmelCase		)

							with open(__UpperCAmelCase      ,					"""rb"""		) as handle:
									lowercase__     			=		pickle.load(__UpperCAmelCase		)

							lowercase__     			=		tokenizer_new.tokenize(__UpperCAmelCase		)

							self.assertListEqual(__UpperCAmelCase      ,					__UpperCAmelCase		)

					@require_sudachi
					def   lowerCamelCase__     (self    :						str		)						->   Optional[int]:




							"""simple docstring"""


							lowercase__     			=		SudachiTokenizer(sudachi_dict_type="""core"""		)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """		)      ,					[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]      ,					)

					@require_sudachi
					def   lowerCamelCase__     (self    :						Union[str, Any]		)						->   Any:




							"""simple docstring"""


							lowercase__     			=		SudachiTokenizer(sudachi_dict_type="""core"""      ,					sudachi_split_mode="""A"""		)

							self.assertListEqual(tokenizer.tokenize("""外国人参政権"""		)      ,					["""外国""", """人""", """参政""", """権"""]		)

					@require_sudachi
					def   lowerCamelCase__     (self    :						str		)						->   Tuple:




							"""simple docstring"""


							lowercase__     			=		SudachiTokenizer(sudachi_dict_type="""core"""      ,					sudachi_split_mode="""B"""		)

							self.assertListEqual(tokenizer.tokenize("""外国人参政権"""		)      ,					["""外国人""", """参政権"""]		)

					@require_sudachi
					def   lowerCamelCase__     (self    :						Optional[Any]		)						->   Tuple:




							"""simple docstring"""


							lowercase__     			=		SudachiTokenizer(sudachi_dict_type="""core"""      ,					sudachi_split_mode="""C"""		)

							self.assertListEqual(tokenizer.tokenize("""外国人参政権"""		)      ,					["""外国人参政権"""]		)

					@require_sudachi
					def   lowerCamelCase__     (self    :						List[Any]		)						->   Dict:




							"""simple docstring"""


							lowercase__     			=		SudachiTokenizer(do_lower_case=__UpperCAmelCase      ,					sudachi_dict_type="""core"""		)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """		)      ,					[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]      ,					)

					@require_sudachi
					def   lowerCamelCase__     (self    :						Union[str, Any]		)						->   Any:




							"""simple docstring"""


							lowercase__     			=		SudachiTokenizer(normalize_text=__UpperCAmelCase      ,					sudachi_dict_type="""core"""		)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """		)      ,					[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]      ,					)

					@require_sudachi
					def   lowerCamelCase__     (self    :						Any		)						->   Optional[Any]:




							"""simple docstring"""


							lowercase__     			=		SudachiTokenizer(trim_whitespace=__UpperCAmelCase      ,					sudachi_dict_type="""core"""		)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """		)      ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,					)

					@require_jumanpp
					def   lowerCamelCase__     (self    :						Optional[Any]		)						->   List[str]:




							"""simple docstring"""


							lowercase__     			=		self.tokenizer_class(self.vocab_file      ,					word_tokenizer_type="""jumanpp"""		)
							self.assertIsNotNone(__UpperCAmelCase		)

							lowercase__     			=		"""こんにちは、世界。\nこんばんは、世界。"""
							lowercase__     			=		tokenizer.tokenize(__UpperCAmelCase		)
							self.assertListEqual(__UpperCAmelCase      ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]		)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase		)      ,					[3, 12, 10, 14, 4, 9, 12, 10, 14]		)

							lowercase__     			=		os.path.join(self.tmpdirname      ,					"""tokenizer.bin"""		)
							with open(__UpperCAmelCase      ,					"""wb"""		) as handle:
									pickle.dump(__UpperCAmelCase      ,					__UpperCAmelCase		)

							with open(__UpperCAmelCase      ,					"""rb"""		) as handle:
									lowercase__     			=		pickle.load(__UpperCAmelCase		)

							lowercase__     			=		tokenizer_new.tokenize(__UpperCAmelCase		)

							self.assertListEqual(__UpperCAmelCase      ,					__UpperCAmelCase		)

					@require_jumanpp
					def   lowerCamelCase__     (self    :						Tuple		)						->   Tuple:




							"""simple docstring"""


							lowercase__     			=		JumanppTokenizer()

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """		)      ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]      ,					)

					@require_jumanpp
					def   lowerCamelCase__     (self    :						int		)						->   Tuple:




							"""simple docstring"""


							lowercase__     			=		JumanppTokenizer(do_lower_case=__UpperCAmelCase		)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """		)      ,					["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]      ,					)

					@require_jumanpp
					def   lowerCamelCase__     (self    :						List[str]		)						->   Tuple:




							"""simple docstring"""


							lowercase__     			=		JumanppTokenizer(normalize_text=__UpperCAmelCase		)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """		)      ,					["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]      ,					)

					@require_jumanpp
					def   lowerCamelCase__     (self    :						int		)						->   Optional[int]:




							"""simple docstring"""


							lowercase__     			=		JumanppTokenizer(trim_whitespace=__UpperCAmelCase		)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """		)      ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]      ,					)

					@require_jumanpp
					def   lowerCamelCase__     (self    :						Union[str, Any]		)						->   Tuple:




							"""simple docstring"""


							lowercase__     			=		JumanppTokenizer()

							self.assertListEqual(
							    tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""		)      ,					["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]      ,					)

					def   lowerCamelCase__     (self    :						List[Any]		)						->   int:




							"""simple docstring"""


							lowercase__     			=		["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

							lowercase__     			=		{}
							for i, token in enumerate(__UpperCAmelCase		):
									lowercase__     			=		i
							lowercase__     			=		WordpieceTokenizer(vocab=__UpperCAmelCase      ,					unk_token="""[UNK]"""		)

							self.assertListEqual(tokenizer.tokenize(""""""		)      ,					[]		)

							self.assertListEqual(tokenizer.tokenize("""こんにちは"""		)      ,					["""こんにちは"""]		)

							self.assertListEqual(tokenizer.tokenize("""こんばんは"""		)      ,					["""こん""", """##ばんは"""]		)

							self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""		)      ,					["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]		)

					def   lowerCamelCase__     (self    :						int		)						->   List[str]:




							"""simple docstring"""


							lowercase__     			=		BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""		)
							lowercase__     			=		tokenizer.subword_tokenizer

							lowercase__     			=		subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""		)
							self.assertListEqual(__UpperCAmelCase      ,					["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]		)

							lowercase__     			=		subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""		)
							self.assertListEqual(__UpperCAmelCase      ,					["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]		)


					def   lowerCamelCase__     (self    :						Union[str, Any]		)						->   List[str]:




							"""simple docstring"""


							lowercase__     			=		self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""		)

							lowercase__     			=		tokenizer.encode("""ありがとう。"""      ,					add_special_tokens=__UpperCAmelCase		)
							lowercase__     			=		tokenizer.encode("""どういたしまして。"""      ,					add_special_tokens=__UpperCAmelCase		)

							lowercase__     			=		tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase		)
							lowercase__     			=		tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase      ,					__UpperCAmelCase		)

							# 2 is for "[CLS]", 3 is for "[SEP]"
							assert encoded_sentence == [2] + text + [3]
							assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class 							A    (       UpperCAmelCase__ ,  unittest.TestCase							):
					'''simple docstring'''






					A__			= BertJapaneseTokenizer
					A__			= False

					def   lowerCamelCase__     (self    :						Union[str, Any]		)						->   int:




							"""simple docstring"""


							super().setUp()

							lowercase__     			=		["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

							lowercase__     			=		os.path.join(self.tmpdirname      ,					VOCAB_FILES_NAMES["""vocab_file"""]		)
							with open(self.vocab_file      ,					"""w"""      ,					encoding="""utf-8"""		) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]		)		)

					def   lowerCamelCase__     (self    :						List[str]      ,					**_UpperCAmelCase    :						str		)						->   List[Any]:




							"""simple docstring"""


							return BertJapaneseTokenizer.from_pretrained(self.tmpdirname      ,					subword_tokenizer_type="""character"""      ,					**__UpperCAmelCase		)

					def   lowerCamelCase__     (self    :						Union[str, Any]      ,					_UpperCAmelCase    :						Optional[Any]		)						->   int:




							"""simple docstring"""


							lowercase__     			=		"""こんにちは、世界。 \nこんばんは、世界。"""
							lowercase__     			=		"""こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
							return input_text, output_text

					def   lowerCamelCase__     (self    :						Any		)						->   Optional[int]:




							"""simple docstring"""


							pass  # TODO add if relevant

					def   lowerCamelCase__     (self    :						Tuple		)						->   int:




							"""simple docstring"""


							pass  # TODO add if relevant

					def   lowerCamelCase__     (self    :						Optional[int]		)						->   Dict:




							"""simple docstring"""


							pass  # TODO add if relevant

					def   lowerCamelCase__     (self    :						Tuple		)						->   Optional[int]:




							"""simple docstring"""


							lowercase__     			=		self.tokenizer_class(self.vocab_file      ,					subword_tokenizer_type="""character"""		)

							lowercase__     			=		tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""		)
							self.assertListEqual(
							    __UpperCAmelCase      ,					["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]		)
							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(__UpperCAmelCase		)      ,					[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]		)

					def   lowerCamelCase__     (self    :						Any		)						->   Optional[int]:




							"""simple docstring"""


							lowercase__     			=		["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

							lowercase__     			=		{}
							for i, token in enumerate(__UpperCAmelCase		):
									lowercase__     			=		i
							lowercase__     			=		CharacterTokenizer(vocab=__UpperCAmelCase      ,					unk_token="""[UNK]"""		)

							self.assertListEqual(tokenizer.tokenize(""""""		)      ,					[]		)

							self.assertListEqual(tokenizer.tokenize("""こんにちは"""		)      ,					["""こ""", """ん""", """に""", """ち""", """は"""]		)

							self.assertListEqual(tokenizer.tokenize("""こんにちほ"""		)      ,					["""こ""", """ん""", """に""", """ち""", """[UNK]"""]		)


					def   lowerCamelCase__     (self    :						Optional[int]		)						->   Dict:




							"""simple docstring"""


							lowercase__     			=		self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""		)

							lowercase__     			=		tokenizer.encode("""ありがとう。"""      ,					add_special_tokens=__UpperCAmelCase		)
							lowercase__     			=		tokenizer.encode("""どういたしまして。"""      ,					add_special_tokens=__UpperCAmelCase		)

							lowercase__     			=		tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase		)
							lowercase__     			=		tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase      ,					__UpperCAmelCase		)

							# 2 is for "[CLS]", 3 is for "[SEP]"
							assert encoded_sentence == [2] + text + [3]
							assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class 							A    (       unittest.TestCase							):
					'''simple docstring'''






					def   lowerCamelCase__     (self    :						Union[str, Any]		)						->   Any:




							"""simple docstring"""


							lowercase__     			=		"""cl-tohoku/bert-base-japanese"""
							lowercase__     			=		AutoTokenizer.from_pretrained(__UpperCAmelCase		)
							self.assertIsInstance(__UpperCAmelCase      ,					__UpperCAmelCase		)






class 							A    (       unittest.TestCase							):
					'''simple docstring'''






					def   lowerCamelCase__     (self    :						List[Any]		)						->   int:




							"""simple docstring"""


							lowercase__     			=		"""cl-tohoku/bert-base-japanese"""
							with self.assertLogs("""transformers"""      ,					level="""WARNING"""		) as cm:
									BertTokenizer.from_pretrained(__UpperCAmelCase		)
									self.assertTrue(
									    cm.records[0].message.startswith(
									        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
									        """ is called from."""		)		)
							lowercase__     			=		"""bert-base-cased"""
							with self.assertLogs("""transformers"""      ,					level="""WARNING"""		) as cm:
									BertJapaneseTokenizer.from_pretrained(__UpperCAmelCase		)
									self.assertTrue(
									    cm.records[0].message.startswith(
									        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
									        """ is called from."""		)		)









import json
import os
import shutil
import tempfile
import unittest

import numpy as np

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES, BertTokenizer
from transformers.testing_utils import require_tokenizers, require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
     from PIL import Image

     from transformers import VisionTextDualEncoderProcessor, ViTImageProcessor






@require_tokenizers
@require_vision
class 		lowercase_					(	unittest.TestCase   ):






      '''simple docstring'''




      def       __lowerCAmelCase					(      self      :						List[Any]  )    ->Tuple:




       """simple docstring"""


       a		  =      tempfile.mkdtemp()

       # fmt: off
       a		  =      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''', ''',''', '''low''', '''lowest''']
       # fmt: on
       a		  =      os.path.join(self.tmpdirname ,					VOCAB_FILES_NAMES['''vocab_file''']  )
       with open(self.vocab_file ,					'''w''' ,					encoding='''utf-8'''  ) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]  )  )

       a		  =      {
           '''do_resize''': True,
           '''size''': {'''height''': 18, '''width''': 18},
           '''do_normalize''': True,
           '''image_mean''': [0.5, 0.5, 0.5],
           '''image_std''': [0.5, 0.5, 0.5],
       }
       a		  =      os.path.join(self.tmpdirname ,					__UpperCAmelCase  )
       with open(self.image_processor_file ,					'''w''' ,					encoding='''utf-8'''  ) as fp:
        json.dump(__UpperCAmelCase ,					__UpperCAmelCase  )







      def       __lowerCAmelCase					(      self      :						List[Any] ,					**__UpperCAmelCase      :						List[Any]  )    ->int:




       """simple docstring"""


       return BertTokenizer.from_pretrained(self.tmpdirname ,					**__UpperCAmelCase  )







      def       __lowerCAmelCase					(      self      :						Union[str, Any] ,					**__UpperCAmelCase      :						Optional[int]  )    ->Union[str, Any]:




       """simple docstring"""


       return ViTImageProcessor.from_pretrained(self.tmpdirname ,					**__UpperCAmelCase  )







      def       __lowerCAmelCase					(      self      :						Tuple  )    ->Any:




       """simple docstring"""


       shutil.rmtree(self.tmpdirname  )







      def       __lowerCAmelCase					(      self      :						str  )    ->Dict:




       """simple docstring"""


       a		  =      [np.random.randint(255 ,					size=(3, 30, 400) ,					dtype=np.uinta  )]

       a		  =      [Image.fromarray(np.moveaxis(__UpperCAmelCase ,					0 ,					-1  )  ) for x in image_inputs]

       return image_inputs







      def       __lowerCAmelCase					(      self      :						Any  )    ->Tuple:




       """simple docstring"""


       a		  =      self.get_tokenizer()
       a		  =      self.get_image_processor()

       a		  =      VisionTextDualEncoderProcessor(tokenizer=__UpperCAmelCase ,					image_processor=__UpperCAmelCase  )

       processor.save_pretrained(self.tmpdirname  )
       a		  =      VisionTextDualEncoderProcessor.from_pretrained(self.tmpdirname  )

       self.assertEqual(processor.tokenizer.get_vocab() ,					tokenizer.get_vocab()  )
       self.assertIsInstance(processor.tokenizer ,					(BertTokenizer, BertTokenizerFast)  )

       self.assertEqual(processor.image_processor.to_json_string() ,					image_processor.to_json_string()  )
       self.assertIsInstance(processor.image_processor ,					__UpperCAmelCase  )







      def       __lowerCAmelCase					(      self      :						int  )    ->Dict:




       """simple docstring"""


       a		  =      VisionTextDualEncoderProcessor(
           tokenizer=self.get_tokenizer() ,					image_processor=self.get_image_processor()  )
       processor.save_pretrained(self.tmpdirname  )

       a		  =      self.get_tokenizer(bos_token='''(BOS)''' ,					eos_token='''(EOS)'''  )
       a		  =      self.get_image_processor(do_normalize=__UpperCAmelCase ,					padding_value=1.0  )

       a		  =      VisionTextDualEncoderProcessor.from_pretrained(
           self.tmpdirname ,					bos_token='''(BOS)''' ,					eos_token='''(EOS)''' ,					do_normalize=__UpperCAmelCase ,					padding_value=1.0  )

       self.assertEqual(processor.tokenizer.get_vocab() ,					tokenizer_add_kwargs.get_vocab()  )
       self.assertIsInstance(processor.tokenizer ,					(BertTokenizer, BertTokenizerFast)  )

       self.assertEqual(processor.image_processor.to_json_string() ,					image_processor_add_kwargs.to_json_string()  )
       self.assertIsInstance(processor.image_processor ,					__UpperCAmelCase  )







      def       __lowerCAmelCase					(      self      :						Tuple  )    ->Dict:




       """simple docstring"""


       a		  =      self.get_image_processor()
       a		  =      self.get_tokenizer()

       a		  =      VisionTextDualEncoderProcessor(tokenizer=__UpperCAmelCase ,					image_processor=__UpperCAmelCase  )

       a		  =      self.prepare_image_inputs()

       a		  =      image_processor(__UpperCAmelCase ,					return_tensors='''np'''  )
       a		  =      processor(images=__UpperCAmelCase ,					return_tensors='''np'''  )

       for key in input_feat_extract.keys():
        self.assertAlmostEqual(input_feat_extract[key].sum() ,					input_processor[key].sum() ,					delta=1e-2  )







      def       __lowerCAmelCase					(      self      :						List[str]  )    ->str:




       """simple docstring"""


       a		  =      self.get_image_processor()
       a		  =      self.get_tokenizer()

       a		  =      VisionTextDualEncoderProcessor(tokenizer=__UpperCAmelCase ,					image_processor=__UpperCAmelCase  )

       a		  =      '''lower newer'''

       a		  =      processor(text=__UpperCAmelCase  )

       a		  =      tokenizer(__UpperCAmelCase  )

       for key in encoded_tok.keys():
        self.assertListEqual(encoded_tok[key] ,					encoded_processor[key]  )







      def       __lowerCAmelCase					(      self      :						List[Any]  )    ->List[str]:




       """simple docstring"""


       a		  =      self.get_image_processor()
       a		  =      self.get_tokenizer()

       a		  =      VisionTextDualEncoderProcessor(tokenizer=__UpperCAmelCase ,					image_processor=__UpperCAmelCase  )

       a		  =      '''lower newer'''
       a		  =      self.prepare_image_inputs()

       a		  =      processor(text=__UpperCAmelCase ,					images=__UpperCAmelCase  )

       self.assertListEqual(list(inputs.keys()  ) ,					['''input_ids''', '''token_type_ids''', '''attention_mask''', '''pixel_values''']  )

       # test if it raises when no input is passed
       with self.assertRaises(__UpperCAmelCase  ):
        processor()







      def       __lowerCAmelCase					(      self      :						Optional[int]  )    ->List[str]:




       """simple docstring"""


       a		  =      self.get_image_processor()
       a		  =      self.get_tokenizer()

       a		  =      VisionTextDualEncoderProcessor(tokenizer=__UpperCAmelCase ,					image_processor=__UpperCAmelCase  )

       a		  =      [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

       a		  =      processor.batch_decode(__UpperCAmelCase  )
       a		  =      tokenizer.batch_decode(__UpperCAmelCase  )

       self.assertListEqual(__UpperCAmelCase ,					__UpperCAmelCase  )







      def       __lowerCAmelCase					(      self      :						Optional[Any]  )    ->Dict:




       """simple docstring"""


       a		  =      self.get_image_processor()
       a		  =      self.get_tokenizer()

       a		  =      VisionTextDualEncoderProcessor(tokenizer=__UpperCAmelCase ,					image_processor=__UpperCAmelCase  )

       a		  =      '''lower newer'''
       a		  =      self.prepare_image_inputs()

       a		  =      processor(text=__UpperCAmelCase ,					images=__UpperCAmelCase  )

       self.assertListEqual(list(inputs.keys()  ) ,					processor.model_input_names  )


