
'''simple docstring'''







import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 						__UpperCAmelCase      (					_lowerCamelCase						,   unittest.TestCase						):
							__lowercase      		=       GPTSanJapaneseTokenizer
							__lowercase      		=       False
							__lowercase      		=       {"""do_clean_text""": False, """add_prefix_space""": False}


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								super().setUp()

								# fmt: off
								_snake_case   						=		['こん', 'こんに', 'にちは', 'ばんは', '世界,㔺界', '、', '。', '<BR>', '<SP>', '<TAB>', '<URL>', '<EMAIL>', '<TEL>', '<DATE>', '<PRICE>', '<BLOCK>', '<KIGOU>', '<U2000U2BFF>', '<|emoji1|>', '<unk>', '<|bagoftoken|>', '<|endoftext|>']
								# fmt: on
								_snake_case   						=		{'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
								_snake_case   						=		{'unk_token': '<unk>'}

								_snake_case   						=		os.path.join(self.tmpdirname    ,   VOCAB_FILES_NAMES['vocab_file']  )
								_snake_case   						=		os.path.join(self.tmpdirname    ,   VOCAB_FILES_NAMES['emoji_file']  )
								with open(self.vocab_file    ,   'w'    ,   encoding='utf-8'  ) as vocab_writer:
									vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )
								with open(self.emoji_file    ,   'w'  ) as emoji_writer:
									emoji_writer.write(json.dumps(lowerCAmelCase_  )  )


							def  lowerCamelCase				(       self    ,   **lowerCAmelCase_  ):





								"""simple docstring"""






								kwargs.update(self.special_tokens_map  )
								return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname    ,   **lowerCAmelCase_  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		'こんにちは、世界。 \nこんばんは、㔺界。😀'
								_snake_case   						=		'こんにちは、世界。 \nこんばんは、世界。😀'
								return input_text, output_text


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case						, _snake_case   						=		self.get_input_output_texts(lowerCAmelCase_  )
								_snake_case   						=		tokenizer.encode(lowerCAmelCase_    ,   add_special_tokens=lowerCAmelCase_  )
								_snake_case   						=		tokenizer.decode(lowerCAmelCase_    ,   clean_up_tokenization_spaces=lowerCAmelCase_  )
								return text, ids


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								pass  # TODO add if relevant


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								pass  # TODO add if relevant


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								pass  # TODO add if relevant


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.get_tokenizer()

								# Testing tokenization
								_snake_case   						=		'こんにちは、世界。　こんばんは、㔺界。'
								_snake_case   						=		['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
								_snake_case   						=		tokenizer.tokenize(lowerCAmelCase_  )
								self.assertListEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )

								# Testing conversion to ids without special tokens
								_snake_case   						=		[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
								_snake_case   						=		tokenizer.convert_tokens_to_ids(lowerCAmelCase_  )
								self.assertListEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )

								# Testing conversion to ids with special tokens
								_snake_case   						=		tokens + [tokenizer.unk_token]
								_snake_case   						=		[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
								_snake_case   						=		tokenizer.convert_tokens_to_ids(lowerCAmelCase_  )
								self.assertListEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.get_tokenizer()

								# Testing tokenization
								_snake_case   						=		'こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'
								_snake_case   						=		'こんにちは、、、、世界。こんばんは、、、、世界。'
								_snake_case   						=		tokenizer.encode(lowerCAmelCase_  )
								_snake_case   						=		tokenizer.decode(lowerCAmelCase_  )
								self.assertEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )


							@slow
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'  )

								# Testing tokenization
								_snake_case   						=		'こんにちは、世界。'
								_snake_case   						=		'こんばんは、㔺界。😀'
								_snake_case   						=		'こんにちは、世界。こんばんは、世界。😀'
								_snake_case   						=		tokenizer.encode(prefix_text + input_text  )
								_snake_case   						=		tokenizer.encode(''    ,   prefix_text=prefix_text + input_text  )
								_snake_case   						=		tokenizer.encode(lowerCAmelCase_    ,   prefix_text=lowerCAmelCase_  )
								_snake_case   						=		tokenizer.decode(lowerCAmelCase_  )
								_snake_case   						=		tokenizer.decode(lowerCAmelCase_  )
								_snake_case   						=		tokenizer.decode(lowerCAmelCase_  )
								self.assertEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )
								self.assertEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )
								self.assertEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )


							@slow
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'  )

								# Testing tokenization
								_snake_case   						=		'こんにちは、世界。'
								_snake_case   						=		'こんばんは、㔺界。😀'

								_snake_case   						=		len(tokenizer.encode(lowerCAmelCase_  )  ) - 2
								_snake_case   						=		len(tokenizer.encode(lowerCAmelCase_  )  ) - 2

								_snake_case   						=		[1] + [0] * (len_prefix + len_text + 1)
								_snake_case   						=		[1] * (len_prefix + len_text + 1) + [0]
								_snake_case   						=		[1] + [1] * (len_prefix) + [0] * (len_text + 1)

								_snake_case   						=		tokenizer(prefix_text + input_text  ).token_type_ids
								_snake_case   						=		tokenizer(''    ,   prefix_text=prefix_text + input_text  ).token_type_ids
								_snake_case   						=		tokenizer(lowerCAmelCase_    ,   prefix_text=lowerCAmelCase_  ).token_type_ids
								self.assertListEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )
								self.assertListEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )
								self.assertListEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )


							@slow
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'  )

								_snake_case   						=		tokenizer.encode('あンいワ'  )
								_snake_case   						=		tokenizer.encode(''    ,   prefix_text='あンいワ'  )
								_snake_case   						=		tokenizer.encode('いワ'    ,   prefix_text='あン'  )

								self.assertEqual(tokenizer.decode(lowerCAmelCase_  )    ,   tokenizer.decode(lowerCAmelCase_  )  )
								self.assertEqual(tokenizer.decode(lowerCAmelCase_  )    ,   tokenizer.decode(lowerCAmelCase_  )  )
								self.assertNotEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )
								self.assertNotEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )
								self.assertEqual(x_token_a[1]    ,   x_token_a[-1]  )  # SEG token
								self.assertEqual(x_token_a[1]    ,   x_token_a[3]  )  # SEG token


							@slow
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'  )

								_snake_case   						=		[['武田信玄', 'は、'], ['織田信長', 'の配下の、']]
								_snake_case   						=		tokenizer(lowerCAmelCase_    ,   padding=lowerCAmelCase_  )
								_snake_case   						=		tokenizer.batch_encode_plus(lowerCAmelCase_    ,   padding=lowerCAmelCase_  )

								# fmt: off
								_snake_case   						=		[[3_59_93, 86_40, 2_59_48, 3_59_98, 3_06_47, 3_56_75, 3_59_99, 3_59_99], [3_59_93, 1_03_82, 98_68, 3_59_98, 3_06_46, 94_59, 3_06_46, 3_56_75]]
								_snake_case   						=		[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
								_snake_case   						=		[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
								# fmt: on
								self.assertListEqual(x_token.input_ids    ,   lowerCAmelCase_  )
								self.assertListEqual(x_token.token_type_ids    ,   lowerCAmelCase_  )
								self.assertListEqual(x_token.attention_mask    ,   lowerCAmelCase_  )
								self.assertListEqual(x_token_a.input_ids    ,   lowerCAmelCase_  )
								self.assertListEqual(x_token_a.token_type_ids    ,   lowerCAmelCase_  )
								self.assertListEqual(x_token_a.attention_mask    ,   lowerCAmelCase_  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								pass




							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								pass



'''simple docstring'''







def 				SCREAMING_SNAKE_CASE__	(   __A	)       ->	bool:
	return number & 1 == 0


if __name__ == "__main__":
					import doctest

					doctest.testmod()

