






import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin




@custom_tokenizers
class 				__A     (			snake_case_      ,     unittest.TestCase ):





				"""simple docstring"""







				UpperCamelCase__ :      Any			 =BertJapaneseTokenizer
				UpperCamelCase__ :      Any			 =False
				UpperCamelCase__ :      Union[str, Any]			 =True


				def       __lowercase							( self    ):




									"""simple docstring"""

									super().setUp()

									__UpperCamelCase      :					Any       		=[
									    '[UNK]',
									    '[CLS]',
									    '[SEP]',
									    'こんにちは',
									    'こん',
									    'にちは',
									    'ばんは',
									    '##こん',
									    '##にちは',
									    '##ばんは',
									    '世界',
									    '##世界',
									    '、',
									    '##、',
									    '。',
									    '##。',
									]

									__UpperCamelCase      :					str       		=os.path.join(self.tmpdirname							,  VOCAB_FILES_NAMES['vocab_file']    )
									with open(self.vocab_file							,  'w'							,  encoding='utf-8'    ) as vocab_writer:
														vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]    )    )


				def       __lowercase							( self							,  lowerCamelCase__    ):




									"""simple docstring"""

									__UpperCamelCase      :					Union[str, Any]       		='こんにちは、世界。 \nこんばんは、世界。'
									__UpperCamelCase      :					Dict       		='こんにちは 、 世界 。 こんばんは 、 世界 。'
									return input_text, output_text


				def       __lowercase							( self							,  lowerCamelCase__    ):




									"""simple docstring"""

									__UpperCamelCase		,				__UpperCamelCase      :					Union[str, Any]       		=self.get_input_output_texts(lowerCamelCase__    )
									__UpperCamelCase      :					str       		=tokenizer.encode(lowerCamelCase__							,  add_special_tokens=lowerCamelCase__    )
									__UpperCamelCase      :					Union[str, Any]       		=tokenizer.decode(lowerCamelCase__							,  clean_up_tokenization_spaces=lowerCamelCase__    )
									return text, ids


				def       __lowercase							( self    ):




									"""simple docstring"""

									pass  # TODO add if relevant


				def       __lowercase							( self    ):




									"""simple docstring"""

									pass  # TODO add if relevant


				def       __lowercase							( self    ):




									"""simple docstring"""

									pass  # TODO add if relevant


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Union[str, Any]       		=self.tokenizer_class(self.vocab_file    )

									__UpperCamelCase      :					int       		=tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'    )
									self.assertListEqual(lowerCamelCase__							,  ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']    )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__    )							,  [3, 12, 10, 14, 4, 9, 12, 10, 14]    )


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					List[Any]       		=self.tokenizer_class(self.vocab_file							,  word_tokenizer_type='mecab'    )
									self.assertIsNotNone(lowerCamelCase__    )

									__UpperCamelCase      :					Optional[Any]       		='こんにちは、世界。\nこんばんは、世界。'
									__UpperCamelCase      :					Union[str, Any]       		=tokenizer.tokenize(lowerCamelCase__    )
									self.assertListEqual(lowerCamelCase__							,  ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']    )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__    )							,  [3, 12, 10, 14, 4, 9, 12, 10, 14]    )

									__UpperCamelCase      :					str       		=os.path.join(self.tmpdirname							,  'tokenizer.bin'    )
									with open(lowerCamelCase__							,  'wb'    ) as handle:
														pickle.dump(lowerCamelCase__							,  lowerCamelCase__    )

									with open(lowerCamelCase__							,  'rb'    ) as handle:
														__UpperCamelCase      :					int       		=pickle.load(lowerCamelCase__    )

									__UpperCamelCase      :					List[Any]       		=tokenizer_new.tokenize(lowerCamelCase__    )

									self.assertListEqual(lowerCamelCase__							,  lowerCamelCase__    )


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Dict       		=MecabTokenizer(mecab_dic='ipadic'    )

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,  ['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']							,  )


				def       __lowercase							( self    ):




									"""simple docstring"""

									try:
														__UpperCamelCase      :					str       		=MecabTokenizer(mecab_dic='unidic_lite'    )
									except ModuleNotFoundError:
														return

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,  ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']							,  )


				def       __lowercase							( self    ):




									"""simple docstring"""

									try:
														__UpperCamelCase      :					List[str]       		=MecabTokenizer(mecab_dic='unidic'    )
									except ModuleNotFoundError:
														return

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,  ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']							,  )


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Union[str, Any]       		=MecabTokenizer(do_lower_case=lowerCamelCase__							,  mecab_dic='ipadic'    )

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,  ['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']							,  )


				def       __lowercase							( self    ):




									"""simple docstring"""

									try:
														__UpperCamelCase      :					Optional[Any]       		=MecabTokenizer(
														    do_lower_case=lowerCamelCase__							,  normalize_text=lowerCamelCase__							,  mecab_option='-d /usr/local/lib/mecab/dic/jumandic'    )
									except RuntimeError:
														# if dict doesn't exist in the system, previous code raises this error.
														return

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,  ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']							,  )


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Optional[Any]       		=MecabTokenizer(normalize_text=lowerCamelCase__							,  mecab_dic='ipadic'    )

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,  ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']							,  )


				@require_sudachi
				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Tuple       		=self.tokenizer_class(self.vocab_file							,  word_tokenizer_type='sudachi'    )
									self.assertIsNotNone(lowerCamelCase__    )

									__UpperCamelCase      :					List[Any]       		='こんにちは、世界。\nこんばんは、世界。'
									__UpperCamelCase      :					Optional[int]       		=tokenizer.tokenize(lowerCamelCase__    )
									self.assertListEqual(lowerCamelCase__							,  ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']    )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__    )							,  [3, 12, 10, 14, 4, 9, 12, 10, 14]    )

									__UpperCamelCase      :					Optional[Any]       		=os.path.join(self.tmpdirname							,  'tokenizer.bin'    )
									with open(lowerCamelCase__							,  'wb'    ) as handle:
														pickle.dump(lowerCamelCase__							,  lowerCamelCase__    )

									with open(lowerCamelCase__							,  'rb'    ) as handle:
														__UpperCamelCase      :					Optional[int]       		=pickle.load(lowerCamelCase__    )

									__UpperCamelCase      :					List[str]       		=tokenizer_new.tokenize(lowerCamelCase__    )

									self.assertListEqual(lowerCamelCase__							,  lowerCamelCase__    )


				@require_sudachi
				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Dict       		=SudachiTokenizer(sudachi_dict_type='core'    )

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,  [' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']							,  )


				@require_sudachi
				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					List[Any]       		=SudachiTokenizer(sudachi_dict_type='core'							,  sudachi_split_mode='A'    )

									self.assertListEqual(tokenizer.tokenize('外国人参政権'    )							,  ['外国', '人', '参政', '権']    )


				@require_sudachi
				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Union[str, Any]       		=SudachiTokenizer(sudachi_dict_type='core'							,  sudachi_split_mode='B'    )

									self.assertListEqual(tokenizer.tokenize('外国人参政権'    )							,  ['外国人', '参政権']    )


				@require_sudachi
				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Optional[Any]       		=SudachiTokenizer(sudachi_dict_type='core'							,  sudachi_split_mode='C'    )

									self.assertListEqual(tokenizer.tokenize('外国人参政権'    )							,  ['外国人参政権']    )


				@require_sudachi
				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					List[str]       		=SudachiTokenizer(do_lower_case=lowerCamelCase__							,  sudachi_dict_type='core'    )

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,  [' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']							,  )


				@require_sudachi
				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Dict       		=SudachiTokenizer(normalize_text=lowerCamelCase__							,  sudachi_dict_type='core'    )

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,  [' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']							,  )


				@require_sudachi
				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Optional[int]       		=SudachiTokenizer(trim_whitespace=lowerCamelCase__							,  sudachi_dict_type='core'    )

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,  ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']							,  )


				@require_jumanpp
				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					int       		=self.tokenizer_class(self.vocab_file							,  word_tokenizer_type='jumanpp'    )
									self.assertIsNotNone(lowerCamelCase__    )

									__UpperCamelCase      :					int       		='こんにちは、世界。\nこんばんは、世界。'
									__UpperCamelCase      :					List[Any]       		=tokenizer.tokenize(lowerCamelCase__    )
									self.assertListEqual(lowerCamelCase__							,  ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']    )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__    )							,  [3, 12, 10, 14, 4, 9, 12, 10, 14]    )

									__UpperCamelCase      :					Dict       		=os.path.join(self.tmpdirname							,  'tokenizer.bin'    )
									with open(lowerCamelCase__							,  'wb'    ) as handle:
														pickle.dump(lowerCamelCase__							,  lowerCamelCase__    )

									with open(lowerCamelCase__							,  'rb'    ) as handle:
														__UpperCamelCase      :					Tuple       		=pickle.load(lowerCamelCase__    )

									__UpperCamelCase      :					Any       		=tokenizer_new.tokenize(lowerCamelCase__    )

									self.assertListEqual(lowerCamelCase__							,  lowerCamelCase__    )


				@require_jumanpp
				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Optional[int]       		=JumanppTokenizer()

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,  ['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']							,  )


				@require_jumanpp
				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					List[Any]       		=JumanppTokenizer(do_lower_case=lowerCamelCase__    )

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,  ['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']							,  )


				@require_jumanpp
				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Optional[Any]       		=JumanppTokenizer(normalize_text=lowerCamelCase__    )

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,  ['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']							,  )


				@require_jumanpp
				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Union[str, Any]       		=JumanppTokenizer(trim_whitespace=lowerCamelCase__    )

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,  ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']							,  )


				@require_jumanpp
				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					int       		=JumanppTokenizer()

									self.assertListEqual(
									    tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'    )							,  ['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']							,  )


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					List[Any]       		=['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

									__UpperCamelCase      :					Any       		={}
									for i, token in enumerate(lowerCamelCase__    ):
														__UpperCamelCase      :					int       		=i
									__UpperCamelCase      :					int       		=WordpieceTokenizer(vocab=lowerCamelCase__							,  unk_token='[UNK]'    )

									self.assertListEqual(tokenizer.tokenize(''    )							,  []    )

									self.assertListEqual(tokenizer.tokenize('こんにちは'    )							,  ['こんにちは']    )

									self.assertListEqual(tokenizer.tokenize('こんばんは'    )							,  ['こん', '##ばんは']    )

									self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'    )							,  ['こん', '##ばんは', '[UNK]', 'こんにちは']    )


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Union[str, Any]       		=BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'    )
									__UpperCamelCase      :					Any       		=tokenizer.subword_tokenizer

									__UpperCamelCase      :					Union[str, Any]       		=subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'    )
									self.assertListEqual(lowerCamelCase__							,  ['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']    )

									__UpperCamelCase      :					str       		=subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'    )
									self.assertListEqual(lowerCamelCase__							,  ['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']    )







				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					List[Any]       		=self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'    )

									__UpperCamelCase      :					Union[str, Any]       		=tokenizer.encode('ありがとう。'							,  add_special_tokens=lowerCamelCase__    )
									__UpperCamelCase      :					int       		=tokenizer.encode('どういたしまして。'							,  add_special_tokens=lowerCamelCase__    )

									__UpperCamelCase      :					str       		=tokenizer.build_inputs_with_special_tokens(lowerCamelCase__    )
									__UpperCamelCase      :					int       		=tokenizer.build_inputs_with_special_tokens(lowerCamelCase__							,  lowerCamelCase__    )

									# 2 is for "[CLS]", 3 is for "[SEP]"
									assert encoded_sentence == [2] + text + [3]
									assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class 				__A     (			snake_case_      ,     unittest.TestCase ):





				"""simple docstring"""







				UpperCamelCase__ :      Any			 =BertJapaneseTokenizer
				UpperCamelCase__ :      Optional[int]			 =False


				def       __lowercase							( self    ):




									"""simple docstring"""

									super().setUp()

									__UpperCamelCase      :					Union[str, Any]       		=['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

									__UpperCamelCase      :					List[Any]       		=os.path.join(self.tmpdirname							,  VOCAB_FILES_NAMES['vocab_file']    )
									with open(self.vocab_file							,  'w'							,  encoding='utf-8'    ) as vocab_writer:
														vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]    )    )


				def       __lowercase							( self							,  **lowerCamelCase__    ):




									"""simple docstring"""

									return BertJapaneseTokenizer.from_pretrained(self.tmpdirname							,  subword_tokenizer_type='character'							,  **lowerCamelCase__    )


				def       __lowercase							( self							,  lowerCamelCase__    ):




									"""simple docstring"""

									__UpperCamelCase      :					Optional[int]       		='こんにちは、世界。 \nこんばんは、世界。'
									__UpperCamelCase      :					str       		='こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
									return input_text, output_text


				def       __lowercase							( self    ):




									"""simple docstring"""

									pass  # TODO add if relevant


				def       __lowercase							( self    ):




									"""simple docstring"""

									pass  # TODO add if relevant


				def       __lowercase							( self    ):




									"""simple docstring"""

									pass  # TODO add if relevant


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Dict       		=self.tokenizer_class(self.vocab_file							,  subword_tokenizer_type='character'    )

									__UpperCamelCase      :					Union[str, Any]       		=tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'    )
									self.assertListEqual(
									    lowerCamelCase__							,  ['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']    )
									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(lowerCamelCase__    )							,  [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]    )


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Any       		=['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

									__UpperCamelCase      :					List[str]       		={}
									for i, token in enumerate(lowerCamelCase__    ):
														__UpperCamelCase      :					List[Any]       		=i
									__UpperCamelCase      :					Dict       		=CharacterTokenizer(vocab=lowerCamelCase__							,  unk_token='[UNK]'    )

									self.assertListEqual(tokenizer.tokenize(''    )							,  []    )

									self.assertListEqual(tokenizer.tokenize('こんにちは'    )							,  ['こ', 'ん', 'に', 'ち', 'は']    )

									self.assertListEqual(tokenizer.tokenize('こんにちほ'    )							,  ['こ', 'ん', 'に', 'ち', '[UNK]']    )







				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					str       		=self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'    )

									__UpperCamelCase      :					List[Any]       		=tokenizer.encode('ありがとう。'							,  add_special_tokens=lowerCamelCase__    )
									__UpperCamelCase      :					Union[str, Any]       		=tokenizer.encode('どういたしまして。'							,  add_special_tokens=lowerCamelCase__    )

									__UpperCamelCase      :					Optional[int]       		=tokenizer.build_inputs_with_special_tokens(lowerCamelCase__    )
									__UpperCamelCase      :					List[Any]       		=tokenizer.build_inputs_with_special_tokens(lowerCamelCase__							,  lowerCamelCase__    )

									# 2 is for "[CLS]", 3 is for "[SEP]"
									assert encoded_sentence == [2] + text + [3]
									assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class 				__A     (			unittest.TestCase ):





				"""simple docstring"""







				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Any       		='cl-tohoku/bert-base-japanese'
									__UpperCamelCase      :					Union[str, Any]       		=AutoTokenizer.from_pretrained(lowerCamelCase__    )
									self.assertIsInstance(lowerCamelCase__							,  lowerCamelCase__    )




class 				__A     (			unittest.TestCase ):





				"""simple docstring"""







				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					List[Any]       		='cl-tohoku/bert-base-japanese'
									with self.assertLogs('transformers'							,  level='WARNING'    ) as cm:
														BertTokenizer.from_pretrained(lowerCamelCase__    )
														self.assertTrue(
														    cm.records[0].message.startswith(
														        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
														        ' is called from.'    )    )
									__UpperCamelCase      :					Union[str, Any]       		='bert-base-cased'
									with self.assertLogs('transformers'							,  level='WARNING'    ) as cm:
														BertJapaneseTokenizer.from_pretrained(lowerCamelCase__    )
														self.assertTrue(
														    cm.records[0].message.startswith(
														        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
														        ' is called from.'    )    )



import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
			    OpenAIGPTConfig,
			    OpenAIGPTDoubleHeadsModel,
			    OpenAIGPTForSequenceClassification,
			    OpenAIGPTLMHeadModel,
			    OpenAIGPTModel,
			)




class     _SCREAMING_SNAKE_CASE     :



	def __init__(    self						,  lowercase						,  lowercase=13						,  lowercase=7						,  lowercase=True						,  lowercase=True						,  lowercase=True						,  lowercase=99						,  lowercase=32						,  lowercase=5						,  lowercase=4						,  lowercase=37						,  lowercase="gelu"						,  lowercase=0.1						,  lowercase=0.1						,  lowercase=512						,  lowercase=16						,  lowercase=2						,  lowercase=0.0_2						,  lowercase=3						,  lowercase=4						,  lowercase=None						,  )				->       List[str]:
					lowerCamelCase_						=			parent
					lowerCamelCase_						=			batch_size
					lowerCamelCase_						=			seq_length
					lowerCamelCase_						=			is_training
					lowerCamelCase_						=			use_token_type_ids
					lowerCamelCase_						=			use_labels
					lowerCamelCase_						=			vocab_size
					lowerCamelCase_						=			hidden_size
					lowerCamelCase_						=			num_hidden_layers
					lowerCamelCase_						=			num_attention_heads
					lowerCamelCase_						=			intermediate_size
					lowerCamelCase_						=			hidden_act
					lowerCamelCase_						=			hidden_dropout_prob
					lowerCamelCase_						=			attention_probs_dropout_prob
					lowerCamelCase_						=			max_position_embeddings
					lowerCamelCase_						=			type_vocab_size
					lowerCamelCase_						=			type_sequence_label_size
					lowerCamelCase_						=			initializer_range
					lowerCamelCase_						=			num_labels
					lowerCamelCase_						=			num_choices
					lowerCamelCase_						=			scope
					lowerCamelCase_						=			self.vocab_size - 1



	def     SCREAMING_SNAKE_CASE_(    self )				->       Any:
					lowerCamelCase_						=			ids_tensor([self.batch_size, self.seq_length]						,  self.vocab_size )

					lowerCamelCase_						=			None
					if self.use_token_type_ids:
									lowerCamelCase_						=			ids_tensor([self.batch_size, self.seq_length]						,  self.type_vocab_size )

					lowerCamelCase_						=			None
					lowerCamelCase_						=			None
					lowerCamelCase_						=			None
					if self.use_labels:
									lowerCamelCase_						=			ids_tensor([self.batch_size]						,  self.type_sequence_label_size )
									lowerCamelCase_						=			ids_tensor([self.batch_size, self.seq_length]						,  self.num_labels )
									lowerCamelCase_						=			ids_tensor([self.batch_size]						,  self.num_choices )

					lowerCamelCase_						=			OpenAIGPTConfig(
					    vocab_size=self.vocab_size						,  n_embd=self.hidden_size						,  n_layer=self.num_hidden_layers						,  n_head=self.num_attention_heads						,  n_positions=self.max_position_embeddings						,  pad_token_id=self.pad_token_id						,  )

					lowerCamelCase_						=			ids_tensor([self.num_hidden_layers, self.num_attention_heads]						,  2 )

					return (
					    config,
					    input_ids,
					    head_mask,
					    token_type_ids,
					    sequence_labels,
					    token_labels,
					    choice_labels,
					)



	def     SCREAMING_SNAKE_CASE_(    self						,  lowercase						,  lowercase						,  lowercase						,  lowercase						,  *lowercase )				->       Dict:
					lowerCamelCase_						=			OpenAIGPTModel(config=lowercase )
					model.to(lowercase )
					model.eval()

					lowerCamelCase_						=			model(lowercase						,  token_type_ids=lowercase						,  head_mask=lowercase )
					lowerCamelCase_						=			model(lowercase						,  token_type_ids=lowercase )
					lowerCamelCase_						=			model(lowercase )

					self.parent.assertEqual(result.last_hidden_state.shape						,  (self.batch_size, self.seq_length, self.hidden_size) )



	def     SCREAMING_SNAKE_CASE_(    self						,  lowercase						,  lowercase						,  lowercase						,  lowercase						,  *lowercase )				->       int:
					lowerCamelCase_						=			OpenAIGPTLMHeadModel(lowercase )
					model.to(lowercase )
					model.eval()

					lowerCamelCase_						=			model(lowercase						,  token_type_ids=lowercase						,  labels=lowercase )
					self.parent.assertEqual(result.loss.shape						,  () )
					self.parent.assertEqual(result.logits.shape						,  (self.batch_size, self.seq_length, self.vocab_size) )



	def     SCREAMING_SNAKE_CASE_(    self						,  lowercase						,  lowercase						,  lowercase						,  lowercase						,  *lowercase )				->       Dict:
					lowerCamelCase_						=			OpenAIGPTDoubleHeadsModel(lowercase )
					model.to(lowercase )
					model.eval()

					lowerCamelCase_						=			model(lowercase						,  token_type_ids=lowercase						,  labels=lowercase )
					self.parent.assertEqual(result.loss.shape						,  () )
					self.parent.assertEqual(result.logits.shape						,  (self.batch_size, self.seq_length, self.vocab_size) )



	def     SCREAMING_SNAKE_CASE_(    self						,  lowercase						,  lowercase						,  lowercase						,  lowercase						,  *lowercase )				->       int:
					lowerCamelCase_						=			self.num_labels
					lowerCamelCase_						=			OpenAIGPTForSequenceClassification(lowercase )
					model.to(lowercase )
					model.eval()

					lowerCamelCase_						=			ids_tensor([self.batch_size]						,  self.type_sequence_label_size )
					lowerCamelCase_						=			model(lowercase						,  token_type_ids=lowercase						,  labels=lowercase )
					self.parent.assertEqual(result.logits.shape						,  (self.batch_size, self.num_labels) )



	def     SCREAMING_SNAKE_CASE_(    self )				->       Union[str, Any]:
					lowerCamelCase_						=			self.prepare_config_and_inputs()
					(
					    (
					    lowerCamelCase_
					)	,    (
					    lowerCamelCase_
					)	,    (
					    lowerCamelCase_
					)	,    (
					    lowerCamelCase_
					)	,    (
					    lowerCamelCase_
					)	,    (
					    lowerCamelCase_
					)	,    (
					    lowerCamelCase_
					)	,    
					)						=			config_and_inputs
					lowerCamelCase_						=			{
					    "input_ids": input_ids,
					    "token_type_ids": token_type_ids,
					    "head_mask": head_mask,
					}

					return config, inputs_dict




@require_torch
class     _SCREAMING_SNAKE_CASE     (      snake_case_			,       snake_case_			,       snake_case_			,       unittest.TestCase						):
	lowerCAmelCase__         = (
	    (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, OpenAIGPTForSequenceClassification)
	    if is_torch_available()
	    else ()
	)
	lowerCAmelCase__         = (
	    (OpenAIGPTLMHeadModel,) if is_torch_available() else ()
	)  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
	lowerCAmelCase__         = (
	    {
	        'feature-extraction': OpenAIGPTModel,
	        'text-classification': OpenAIGPTForSequenceClassification,
	        'text-generation': OpenAIGPTLMHeadModel,
	        'zero-shot': OpenAIGPTForSequenceClassification,
	    }
	    if is_torch_available()
	    else {}
	)



	def     SCREAMING_SNAKE_CASE_(    self						,  lowercase						,  lowercase						,  lowercase						,  lowercase						,  lowercase )				->       int:
					if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
									# Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
									# `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
									# tiny config could not be created.
									return True

					return False



	def     SCREAMING_SNAKE_CASE_(    self						,  lowercase						,  lowercase						,  lowercase=False )				->       Any:
					lowerCamelCase_						=			super()._prepare_for_class(lowercase						,  lowercase						,  return_labels=lowercase )

					if return_labels:
									if model_class.__name__ == "OpenAIGPTDoubleHeadsModel":
													lowerCamelCase_						=			torch.zeros(
													    (self.model_tester.batch_size, self.model_tester.num_choices, self.model_tester.seq_length)						,  dtype=torch.long						,  device=lowercase						,  )
													lowerCamelCase_						=			inputs_dict["labels"]
													lowerCamelCase_						=			inputs_dict["labels"]
													lowerCamelCase_						=			torch.zeros(
													    (self.model_tester.batch_size, self.model_tester.num_choices)						,  dtype=torch.long						,  device=lowercase						,  )
													lowerCamelCase_						=			torch.zeros(
													    self.model_tester.batch_size						,  dtype=torch.long						,  device=lowercase )
					return inputs_dict



	def     SCREAMING_SNAKE_CASE_(    self )				->       Dict:
					lowerCamelCase_						=			OpenAIGPTModelTester(self )
					lowerCamelCase_						=			ConfigTester(self						,  config_class=lowercase						,  n_embd=37 )



	def     SCREAMING_SNAKE_CASE_(    self )				->       Any:
					self.config_tester.run_common_tests()



	def     SCREAMING_SNAKE_CASE_(    self )				->       Union[str, Any]:
					lowerCamelCase_						=			self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_openai_gpt_model(*lowercase )



	def     SCREAMING_SNAKE_CASE_(    self )				->       str:
					lowerCamelCase_						=			self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_lm_head_model(*lowercase )



	def     SCREAMING_SNAKE_CASE_(    self )				->       Tuple:
					lowerCamelCase_						=			self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_double_lm_head_model(*lowercase )



	def     SCREAMING_SNAKE_CASE_(    self )				->       Optional[Any]:
					lowerCamelCase_						=			self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*lowercase )



	@slow
	def     SCREAMING_SNAKE_CASE_(    self )				->       Any:
					for model_name in OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									lowerCamelCase_						=			OpenAIGPTModel.from_pretrained(lowercase )
									self.assertIsNotNone(lowercase )




@require_torch
class     _SCREAMING_SNAKE_CASE     (      unittest.TestCase						):



	@slow
	def     SCREAMING_SNAKE_CASE_(    self )				->       str:
					lowerCamelCase_						=			OpenAIGPTLMHeadModel.from_pretrained("openai-gpt" )
					model.to(lowercase )
					lowerCamelCase_						=			torch.tensor([[481, 4735, 544]]						,  dtype=torch.long						,  device=lowercase )  # the president is
					lowerCamelCase_						=			[
					    481,
					    4735,
					    544,
					    246,
					    963,
					    870,
					    762,
					    239,
					    244,
					    40477,
					    244,
					    249,
					    719,
					    881,
					    487,
					    544,
					    240,
					    244,
					    603,
					    481,
					]  # the president is a very good man. " \n " i\'m sure he is, " said the

					lowerCamelCase_						=			model.generate(lowercase						,  do_sample=lowercase )
					self.assertListEqual(output_ids[0].tolist()						,  lowercase )


