






import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin



@custom_tokenizers
class   lowerCamelCase      (_snake_case ,  unittest.TestCase			):







		'''simple docstring'''



		_snake_case    :    str       						=  BertJapaneseTokenizer
		_snake_case    :    Dict       						=  False
		_snake_case    :    int       						=  True



		def 					__UpperCAmelCase			(							self   )	->							str:
							super().setUp()

							UpperCAmelCase_    :      str            =						[
							    '[UNK]',
							    '[CLS]',
							    '[SEP]',
							    'こんにちは',
							    'こん',
							    'にちは',
							    'ばんは',
							    '##こん',
							    '##にちは',
							    '##ばんは',
							    '世界',
							    '##世界',
							    '、',
							    '##、',
							    '。',
							    '##。',
							]

							UpperCAmelCase_    :      str            =						os.path.join(self.tmpdirname				,		VOCAB_FILES_NAMES['vocab_file']   )
							with open(self.vocab_file				,		'w'				,		encoding='utf-8'   ) as vocab_writer:
												vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]   )   )



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase   )	->							Dict:
							UpperCAmelCase_    :      List[Any]            =						'こんにちは、世界。 \nこんばんは、世界。'
							UpperCAmelCase_    :      List[Any]            =						'こんにちは 、 世界 。 こんばんは 、 世界 。'
							return input_text, output_text



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase   )	->							Union[str, Any]:
							UpperCAmelCase_       ,  UpperCAmelCase_    :      int            =						self.get_input_output_texts(_UpperCamelCase   )
							UpperCAmelCase_    :      int            =						tokenizer.encode(_UpperCamelCase				,		add_special_tokens=_UpperCamelCase   )
							UpperCAmelCase_    :      Tuple            =						tokenizer.decode(_UpperCamelCase				,		clean_up_tokenization_spaces=_UpperCamelCase   )
							return text, ids



		def 					__UpperCAmelCase			(							self   )	->							Dict:
							pass  # TODO add if relevant



		def 					__UpperCAmelCase			(							self   )	->							Optional[Any]:
							pass  # TODO add if relevant



		def 					__UpperCAmelCase			(							self   )	->							Optional[Any]:
							pass  # TODO add if relevant



		def 					__UpperCAmelCase			(							self   )	->							str:
							UpperCAmelCase_    :      Any            =						self.tokenizer_class(self.vocab_file   )

							UpperCAmelCase_    :      Dict            =						tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'   )
							self.assertListEqual(_UpperCamelCase				,		['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']   )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase   )				,		[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]   )



		def 					__UpperCAmelCase			(							self   )	->							Tuple:
							UpperCAmelCase_    :      Tuple            =						self.tokenizer_class(self.vocab_file				,		word_tokenizer_type='mecab'   )
							self.assertIsNotNone(_UpperCamelCase   )

							UpperCAmelCase_    :      List[str]            =						'こんにちは、世界。\nこんばんは、世界。'
							UpperCAmelCase_    :      List[Any]            =						tokenizer.tokenize(_UpperCamelCase   )
							self.assertListEqual(_UpperCamelCase				,		['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']   )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase   )				,		[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]   )

							UpperCAmelCase_    :      Optional[Any]            =						os.path.join(self.tmpdirname				,		'tokenizer.bin'   )
							with open(_UpperCamelCase				,		'wb'   ) as handle:
												pickle.dump(_UpperCamelCase				,		_UpperCamelCase   )

							with open(_UpperCamelCase				,		'rb'   ) as handle:
												UpperCAmelCase_    :      str            =						pickle.load(_UpperCamelCase   )

							UpperCAmelCase_    :      Union[str, Any]            =						tokenizer_new.tokenize(_UpperCamelCase   )

							self.assertListEqual(_UpperCamelCase				,		_UpperCamelCase   )



		def 					__UpperCAmelCase			(							self   )	->							List[Any]:
							UpperCAmelCase_    :      Optional[Any]            =						MecabTokenizer(mecab_dic='ipadic'   )

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '   )				,		['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']				,		)



		def 					__UpperCAmelCase			(							self   )	->							Union[str, Any]:
							try:
												UpperCAmelCase_    :      List[str]            =						MecabTokenizer(mecab_dic='unidic_lite'   )
							except ModuleNotFoundError:
												return

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '   )				,		['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']				,		)



		def 					__UpperCAmelCase			(							self   )	->							Optional[Any]:
							try:
												UpperCAmelCase_    :      Optional[int]            =						MecabTokenizer(mecab_dic='unidic'   )
							except ModuleNotFoundError:
												return

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '   )				,		['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']				,		)



		def 					__UpperCAmelCase			(							self   )	->							Dict:
							UpperCAmelCase_    :      Dict            =						MecabTokenizer(do_lower_case=_UpperCamelCase				,		mecab_dic='ipadic'   )

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '   )				,		['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']				,		)



		def 					__UpperCAmelCase			(							self   )	->							str:
							try:
												UpperCAmelCase_    :      int            =						MecabTokenizer(
												    do_lower_case=_UpperCamelCase				,		normalize_text=_UpperCamelCase				,		mecab_option='-d /usr/local/lib/mecab/dic/jumandic'   )
							except RuntimeError:
												# if dict doesn't exist in the system, previous code raises this error.
												return

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '   )				,		['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']				,		)



		def 					__UpperCAmelCase			(							self   )	->							Tuple:
							UpperCAmelCase_    :      Any            =						MecabTokenizer(normalize_text=_UpperCamelCase				,		mecab_dic='ipadic'   )

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '   )				,		['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']				,		)



		@require_sudachi
		def 					__UpperCAmelCase			(							self   )	->							Optional[int]:
							UpperCAmelCase_    :      Dict            =						self.tokenizer_class(self.vocab_file				,		word_tokenizer_type='sudachi'   )
							self.assertIsNotNone(_UpperCamelCase   )

							UpperCAmelCase_    :      Any            =						'こんにちは、世界。\nこんばんは、世界。'
							UpperCAmelCase_    :      List[Any]            =						tokenizer.tokenize(_UpperCamelCase   )
							self.assertListEqual(_UpperCamelCase				,		['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']   )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase   )				,		[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]   )

							UpperCAmelCase_    :      Dict            =						os.path.join(self.tmpdirname				,		'tokenizer.bin'   )
							with open(_UpperCamelCase				,		'wb'   ) as handle:
												pickle.dump(_UpperCamelCase				,		_UpperCamelCase   )

							with open(_UpperCamelCase				,		'rb'   ) as handle:
												UpperCAmelCase_    :      Tuple            =						pickle.load(_UpperCamelCase   )

							UpperCAmelCase_    :      Optional[int]            =						tokenizer_new.tokenize(_UpperCamelCase   )

							self.assertListEqual(_UpperCamelCase				,		_UpperCamelCase   )



		@require_sudachi
		def 					__UpperCAmelCase			(							self   )	->							str:
							UpperCAmelCase_    :      Tuple            =						SudachiTokenizer(sudachi_dict_type='core'   )

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '   )				,		[' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']				,		)



		@require_sudachi
		def 					__UpperCAmelCase			(							self   )	->							List[str]:
							UpperCAmelCase_    :      Union[str, Any]            =						SudachiTokenizer(sudachi_dict_type='core'				,		sudachi_split_mode='A'   )

							self.assertListEqual(tokenizer.tokenize('外国人参政権'   )				,		['外国', '人', '参政', '権']   )



		@require_sudachi
		def 					__UpperCAmelCase			(							self   )	->							List[Any]:
							UpperCAmelCase_    :      Optional[int]            =						SudachiTokenizer(sudachi_dict_type='core'				,		sudachi_split_mode='B'   )

							self.assertListEqual(tokenizer.tokenize('外国人参政権'   )				,		['外国人', '参政権']   )



		@require_sudachi
		def 					__UpperCAmelCase			(							self   )	->							Union[str, Any]:
							UpperCAmelCase_    :      List[Any]            =						SudachiTokenizer(sudachi_dict_type='core'				,		sudachi_split_mode='C'   )

							self.assertListEqual(tokenizer.tokenize('外国人参政権'   )				,		['外国人参政権']   )



		@require_sudachi
		def 					__UpperCAmelCase			(							self   )	->							Any:
							UpperCAmelCase_    :      Tuple            =						SudachiTokenizer(do_lower_case=_UpperCamelCase				,		sudachi_dict_type='core'   )

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '   )				,		[' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']				,		)



		@require_sudachi
		def 					__UpperCAmelCase			(							self   )	->							Tuple:
							UpperCAmelCase_    :      List[str]            =						SudachiTokenizer(normalize_text=_UpperCamelCase				,		sudachi_dict_type='core'   )

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '   )				,		[' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']				,		)



		@require_sudachi
		def 					__UpperCAmelCase			(							self   )	->							str:
							UpperCAmelCase_    :      List[Any]            =						SudachiTokenizer(trim_whitespace=_UpperCamelCase				,		sudachi_dict_type='core'   )

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '   )				,		['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']				,		)



		@require_jumanpp
		def 					__UpperCAmelCase			(							self   )	->							Any:
							UpperCAmelCase_    :      Dict            =						self.tokenizer_class(self.vocab_file				,		word_tokenizer_type='jumanpp'   )
							self.assertIsNotNone(_UpperCamelCase   )

							UpperCAmelCase_    :      Union[str, Any]            =						'こんにちは、世界。\nこんばんは、世界。'
							UpperCAmelCase_    :      Optional[Any]            =						tokenizer.tokenize(_UpperCamelCase   )
							self.assertListEqual(_UpperCamelCase				,		['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']   )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase   )				,		[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]   )

							UpperCAmelCase_    :      int            =						os.path.join(self.tmpdirname				,		'tokenizer.bin'   )
							with open(_UpperCamelCase				,		'wb'   ) as handle:
												pickle.dump(_UpperCamelCase				,		_UpperCamelCase   )

							with open(_UpperCamelCase				,		'rb'   ) as handle:
												UpperCAmelCase_    :      str            =						pickle.load(_UpperCamelCase   )

							UpperCAmelCase_    :      Dict            =						tokenizer_new.tokenize(_UpperCamelCase   )

							self.assertListEqual(_UpperCamelCase				,		_UpperCamelCase   )



		@require_jumanpp
		def 					__UpperCAmelCase			(							self   )	->							Tuple:
							UpperCAmelCase_    :      Union[str, Any]            =						JumanppTokenizer()

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '   )				,		['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']				,		)



		@require_jumanpp
		def 					__UpperCAmelCase			(							self   )	->							int:
							UpperCAmelCase_    :      int            =						JumanppTokenizer(do_lower_case=_UpperCamelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '   )				,		['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']				,		)



		@require_jumanpp
		def 					__UpperCAmelCase			(							self   )	->							Union[str, Any]:
							UpperCAmelCase_    :      Any            =						JumanppTokenizer(normalize_text=_UpperCamelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '   )				,		['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']				,		)



		@require_jumanpp
		def 					__UpperCAmelCase			(							self   )	->							List[str]:
							UpperCAmelCase_    :      List[Any]            =						JumanppTokenizer(trim_whitespace=_UpperCamelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '   )				,		['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']				,		)



		@require_jumanpp
		def 					__UpperCAmelCase			(							self   )	->							Union[str, Any]:
							UpperCAmelCase_    :      Dict            =						JumanppTokenizer()

							self.assertListEqual(
							    tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'   )				,		['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']				,		)



		def 					__UpperCAmelCase			(							self   )	->							List[str]:
							UpperCAmelCase_    :      Optional[Any]            =						['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

							UpperCAmelCase_    :      Any            =						{}
							for i, token in enumerate(_UpperCamelCase   ):
												UpperCAmelCase_    :      Union[str, Any]            =						i
							UpperCAmelCase_    :      List[Any]            =						WordpieceTokenizer(vocab=_UpperCamelCase				,		unk_token='[UNK]'   )

							self.assertListEqual(tokenizer.tokenize(''   )				,		[]   )

							self.assertListEqual(tokenizer.tokenize('こんにちは'   )				,		['こんにちは']   )

							self.assertListEqual(tokenizer.tokenize('こんばんは'   )				,		['こん', '##ばんは']   )

							self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'   )				,		['こん', '##ばんは', '[UNK]', 'こんにちは']   )



		def 					__UpperCAmelCase			(							self   )	->							str:
							UpperCAmelCase_    :      Union[str, Any]            =						BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'   )
							UpperCAmelCase_    :      str            =						tokenizer.subword_tokenizer

							UpperCAmelCase_    :      Union[str, Any]            =						subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'   )
							self.assertListEqual(_UpperCamelCase				,		['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']   )

							UpperCAmelCase_    :      Any            =						subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'   )
							self.assertListEqual(_UpperCamelCase				,		['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']   )



		def 					__UpperCAmelCase			(							self   )	->							Tuple:
							UpperCAmelCase_    :      Optional[Any]            =						self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'   )

							UpperCAmelCase_    :      Optional[Any]            =						tokenizer.encode('ありがとう。'				,		add_special_tokens=_UpperCamelCase   )
							UpperCAmelCase_    :      str            =						tokenizer.encode('どういたしまして。'				,		add_special_tokens=_UpperCamelCase   )

							UpperCAmelCase_    :      int            =						tokenizer.build_inputs_with_special_tokens(_UpperCamelCase   )
							UpperCAmelCase_    :      Dict            =						tokenizer.build_inputs_with_special_tokens(_UpperCamelCase				,		_UpperCamelCase   )

							# 2 is for "[CLS]", 3 is for "[SEP]"
							assert encoded_sentence == [2] + text + [3]
							assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class   lowerCamelCase      (_snake_case ,  unittest.TestCase			):







		'''simple docstring'''



		_snake_case    :    Dict       						=  BertJapaneseTokenizer
		_snake_case    :    Any       						=  False



		def 					__UpperCAmelCase			(							self   )	->							Tuple:
							super().setUp()

							UpperCAmelCase_    :      List[str]            =						['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

							UpperCAmelCase_    :      int            =						os.path.join(self.tmpdirname				,		VOCAB_FILES_NAMES['vocab_file']   )
							with open(self.vocab_file				,		'w'				,		encoding='utf-8'   ) as vocab_writer:
												vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]   )   )



		def 					__UpperCAmelCase			(							self				,		**_UpperCamelCase   )	->							Optional[int]:
							return BertJapaneseTokenizer.from_pretrained(self.tmpdirname				,		subword_tokenizer_type='character'				,		**_UpperCamelCase   )



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase   )	->							Tuple:
							UpperCAmelCase_    :      Union[str, Any]            =						'こんにちは、世界。 \nこんばんは、世界。'
							UpperCAmelCase_    :      Optional[int]            =						'こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
							return input_text, output_text



		def 					__UpperCAmelCase			(							self   )	->							Dict:
							pass  # TODO add if relevant



		def 					__UpperCAmelCase			(							self   )	->							List[str]:
							pass  # TODO add if relevant



		def 					__UpperCAmelCase			(							self   )	->							str:
							pass  # TODO add if relevant



		def 					__UpperCAmelCase			(							self   )	->							List[str]:
							UpperCAmelCase_    :      Optional[Any]            =						self.tokenizer_class(self.vocab_file				,		subword_tokenizer_type='character'   )

							UpperCAmelCase_    :      Optional[Any]            =						tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'   )
							self.assertListEqual(
							    _UpperCamelCase				,		['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']   )
							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(_UpperCamelCase   )				,		[3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]   )



		def 					__UpperCAmelCase			(							self   )	->							Optional[int]:
							UpperCAmelCase_    :      List[str]            =						['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

							UpperCAmelCase_    :      Dict            =						{}
							for i, token in enumerate(_UpperCamelCase   ):
												UpperCAmelCase_    :      Any            =						i
							UpperCAmelCase_    :      int            =						CharacterTokenizer(vocab=_UpperCamelCase				,		unk_token='[UNK]'   )

							self.assertListEqual(tokenizer.tokenize(''   )				,		[]   )

							self.assertListEqual(tokenizer.tokenize('こんにちは'   )				,		['こ', 'ん', 'に', 'ち', 'は']   )

							self.assertListEqual(tokenizer.tokenize('こんにちほ'   )				,		['こ', 'ん', 'に', 'ち', '[UNK]']   )



		def 					__UpperCAmelCase			(							self   )	->							Union[str, Any]:
							UpperCAmelCase_    :      List[Any]            =						self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'   )

							UpperCAmelCase_    :      Dict            =						tokenizer.encode('ありがとう。'				,		add_special_tokens=_UpperCamelCase   )
							UpperCAmelCase_    :      Dict            =						tokenizer.encode('どういたしまして。'				,		add_special_tokens=_UpperCamelCase   )

							UpperCAmelCase_    :      Any            =						tokenizer.build_inputs_with_special_tokens(_UpperCamelCase   )
							UpperCAmelCase_    :      Dict            =						tokenizer.build_inputs_with_special_tokens(_UpperCamelCase				,		_UpperCamelCase   )

							# 2 is for "[CLS]", 3 is for "[SEP]"
							assert encoded_sentence == [2] + text + [3]
							assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class   lowerCamelCase      (unittest.TestCase			):







		'''simple docstring'''



		def 					__UpperCAmelCase			(							self   )	->							Optional[int]:
							UpperCAmelCase_    :      List[Any]            =						'cl-tohoku/bert-base-japanese'
							UpperCAmelCase_    :      List[str]            =						AutoTokenizer.from_pretrained(_UpperCamelCase   )
							self.assertIsInstance(_UpperCamelCase				,		_UpperCamelCase   )





class   lowerCamelCase      (unittest.TestCase			):







		'''simple docstring'''



		def 					__UpperCAmelCase			(							self   )	->							Union[str, Any]:
							UpperCAmelCase_    :      Tuple            =						'cl-tohoku/bert-base-japanese'
							with self.assertLogs('transformers'				,		level='WARNING'   ) as cm:
												BertTokenizer.from_pretrained(_UpperCamelCase   )
												self.assertTrue(
												    cm.records[0].message.startswith(
												        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
												        ' is called from.'   )   )
							UpperCAmelCase_    :      List[Any]            =						'bert-base-cased'
							with self.assertLogs('transformers'				,		level='WARNING'   ) as cm:
												BertJapaneseTokenizer.from_pretrained(_UpperCamelCase   )
												self.assertTrue(
												    cm.records[0].message.startswith(
												        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
												        ' is called from.'   )   )














import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


__UpperCAmelCase								=						logging.get_logger(__name__)

__UpperCAmelCase								=						{'vocab_file': 'vocab.txt'}

__UpperCAmelCase								=						{
    'vocab_file': {
        'YituTech/conv-bert-base': 'https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt',
        'YituTech/conv-bert-medium-small': (
            'https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt'
        ),
        'YituTech/conv-bert-small': 'https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt',
    }
}

__UpperCAmelCase								=						{
    'YituTech/conv-bert-base': 512,
    'YituTech/conv-bert-medium-small': 512,
    'YituTech/conv-bert-small': 512,
}


__UpperCAmelCase								=						{
    'YituTech/conv-bert-base': {'do_lower_case': True},
    'YituTech/conv-bert-medium-small': {'do_lower_case': True},
    'YituTech/conv-bert-small': {'do_lower_case': True},
}



class   lowerCamelCase      (_snake_case			):







		'''simple docstring'''



		_snake_case    :    Optional[int]       						=  VOCAB_FILES_NAMES
		_snake_case    :    int       						=  PRETRAINED_VOCAB_FILES_MAP
		_snake_case    :    Dict       						=  PRETRAINED_INIT_CONFIGURATION
		_snake_case    :    List[str]       						=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_snake_case    :    Any       						=  ConvBertTokenizer



		def __init__(							self				,		_UpperCamelCase=None				,		_UpperCamelCase=None				,		_UpperCamelCase=True				,		_UpperCamelCase="[UNK]"				,		_UpperCamelCase="[SEP]"				,		_UpperCamelCase="[PAD]"				,		_UpperCamelCase="[CLS]"				,		_UpperCamelCase="[MASK]"				,		_UpperCamelCase=True				,		_UpperCamelCase=None				,		**_UpperCamelCase				,		)	->							Dict:
							super().__init__(
							    _UpperCamelCase				,		tokenizer_file=_UpperCamelCase				,		do_lower_case=_UpperCamelCase				,		unk_token=_UpperCamelCase				,		sep_token=_UpperCamelCase				,		pad_token=_UpperCamelCase				,		cls_token=_UpperCamelCase				,		mask_token=_UpperCamelCase				,		tokenize_chinese_chars=_UpperCamelCase				,		strip_accents=_UpperCamelCase				,		**_UpperCamelCase				,		)

							UpperCAmelCase_    :      Optional[int]            =						json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
							if (
							    normalizer_state.get('lowercase'				,		_UpperCamelCase   ) != do_lower_case
							    or normalizer_state.get('strip_accents'				,		_UpperCamelCase   ) != strip_accents
							    or normalizer_state.get('handle_chinese_chars'				,		_UpperCamelCase   ) != tokenize_chinese_chars
							):
												UpperCAmelCase_    :      Any            =						getattr(_UpperCamelCase				,		normalizer_state.pop('type'   )   )
												UpperCAmelCase_    :      str            =						do_lower_case
												UpperCAmelCase_    :      List[Any]            =						strip_accents
												UpperCAmelCase_    :      str            =						tokenize_chinese_chars
												UpperCAmelCase_    :      Tuple            =						normalizer_class(**_UpperCamelCase   )

							UpperCAmelCase_    :      Any            =						do_lower_case



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase=None   )	->							List[str]:
							UpperCAmelCase_    :      int            =						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
												output += token_ids_a + [self.sep_token_id]

							return output



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase = None   )	->							List[int]:
							UpperCAmelCase_    :      Union[str, Any]            =						[self.sep_token_id]
							UpperCAmelCase_    :      int            =						[self.cls_token_id]
							if token_ids_a is None:
												return len(cls + token_ids_a + sep   ) * [0]
							return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase = None   )	->							Tuple[str]:
							UpperCAmelCase_    :      Any            =						self._tokenizer.model.save(_UpperCamelCase				,		name=_UpperCamelCase   )
							return tuple(_UpperCamelCase   )






