





'''simple docstring'''






import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 	_lowercase    (				__UpperCamelCase		,       unittest.TestCase					):
   _UpperCamelCase									= GPTNeoXJapaneseTokenizer
   _UpperCamelCase									= False
   _UpperCamelCase									= {"""do_clean_text""": False, """add_prefix_space""": False}


   def        snake_case					(			self       ):
       super().setUp()

       A							:							Optional[int]				=    [
           """こん""",
           """こんに""",
           """にちは""",
           """ばんは""",
           """世界,㔺界""",
           """、""",
           """。""",
           """<BR>""",
           """<SP>""",
           """<TAB>""",
           """<URL>""",
           """<EMAIL>""",
           """<TEL>""",
           """<DATE>""",
           """<PRICE>""",
           """<BLOCK>""",
           """<KIGOU>""",
           """<U2000U2BFF>""",
           """<|emoji1|>""",
           """<unk>""",
           """<|startoftext|>""",
           """<|endoftext|>""",
       ]
       A							:							Any				=    {"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
       A							:							List[str]				=    {"""unk_token""": """<unk>"""}

       A							:							Any				=    os.path.join(self.tmpdirname     , VOCAB_FILES_NAMES['''vocab_file''']       )
       A							:							Tuple				=    os.path.join(self.tmpdirname     , VOCAB_FILES_NAMES['''emoji_file''']       )
       with open(self.vocab_file     , '''w'''     , encoding='''utf-8'''       ) as vocab_writer:
           vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]       )       )
       with open(self.emoji_file     , '''w'''       ) as emoji_writer:
           emoji_writer.write(json.dumps(__UpperCamelCase       )       )


   def        snake_case					(			self     , **_UpperCAmelCase       ):
       kwargs.update(self.special_tokens_map       )
       return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname     , **__UpperCamelCase       )


   def        snake_case					(			self     , _UpperCAmelCase       ):
       A							:							Any				=    """こんにちは、世界。 \nこんばんは、㔺界。😀"""
       A							:							Optional[int]				=    """こんにちは、世界。 \nこんばんは、世界。😀"""
       return input_text, output_text


   def        snake_case					(			self     , _UpperCAmelCase       ):
       A							:							List[str]				=    self.get_input_output_texts(__UpperCamelCase       )
       A							:							List[str]				=    tokenizer.encode(__UpperCamelCase     , add_special_tokens=__UpperCamelCase       )
       A							:							str				=    tokenizer.decode(__UpperCamelCase     , clean_up_tokenization_spaces=__UpperCamelCase       )
       return text, ids


   def        snake_case					(			self       ):
       pass  # TODO add if relevant


   def        snake_case					(			self       ):
       pass  # TODO add if relevant


   def        snake_case					(			self       ):
       pass  # TODO add if relevant


   def        snake_case					(			self       ):
       A							:							List[Any]				=    self.get_tokenizer()

       # Testing tokenization
       A							:							Dict				=    """こんにちは、世界。　こんばんは、㔺界。"""
       A							:							Optional[int]				=    ["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
       A							:							str				=    tokenizer.tokenize(__UpperCamelCase       )
       self.assertListEqual(__UpperCamelCase     , __UpperCamelCase       )

       # Testing conversion to ids without special tokens
       A							:							Optional[Any]				=    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
       A							:							Optional[int]				=    tokenizer.convert_tokens_to_ids(__UpperCamelCase       )
       self.assertListEqual(__UpperCamelCase     , __UpperCamelCase       )

       # Testing conversion to ids with special tokens
       A							:							List[str]				=    tokens + [tokenizer.unk_token]
       A							:							Dict				=    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
       A							:							Any				=    tokenizer.convert_tokens_to_ids(__UpperCamelCase       )
       self.assertListEqual(__UpperCamelCase     , __UpperCamelCase       )


   @slow
   def        snake_case					(			self       ):
       A							:							int				=    self.tokenizer_class.from_pretrained('''abeja/gpt-neox-japanese-2.7b'''       )

       A							:							Union[str, Any]				=    tokenizer.encode('''ありがとう。'''     , add_special_tokens=__UpperCamelCase       )
       A							:							Dict				=    tokenizer.encode('''どういたしまして。'''     , add_special_tokens=__UpperCamelCase       )

       A							:							Optional[Any]				=    tokenizer.build_inputs_with_special_tokens(__UpperCamelCase       )
       A							:							int				=    tokenizer.build_inputs_with_special_tokens(__UpperCamelCase     , __UpperCamelCase       )

       assert encoded_sentence == ids_a
       assert encoded_pair == ids_a + ids_a


   def        snake_case					(			self       ):
       pass


   def        snake_case					(			self       ):
       pass


'''simple docstring'''







import unittest

from cer import CER


__SCREAMING_SNAKE_CASE		:  List[str]        =      CER()
class  SCREAMING_SNAKE_CASE__       (  unittest.TestCase   ):
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     Optional[Any]								=	["""White House"""]
					__a       :     Tuple								=	["""white house"""]
					# S = 2, D = 0, I = 0, N = 11, CER = 2 / 11
					__a       :     Any								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.1_8_1_8_1_8_1_8_1_8			) < 1E-6			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     Optional[int]								=	["""were wolf"""]
					__a       :     Tuple								=	["""werewolf"""]
					# S = 0, D = 0, I = 1, N = 9, CER = 1 / 9
					__a       :     Any								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.1_1_1_1_1_1_1			) < 1E-6			)

					__a       :     str								=	["""werewolf"""]
					__a       :     List[Any]								=	["""weae     wolf"""]
					# S = 1, D = 1, I = 0, N = 8, CER = 0.25
					__a       :     Tuple								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.2_5			) < 1E-6			)

					# consecutive whitespaces case 1
					__a       :     int								=	["""were wolf"""]
					__a       :     Dict								=	["""were               wolf"""]
					# S = 0, D = 0, I = 0, N = 9, CER = 0
					__a       :     str								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.0			) < 1E-6			)

					# consecutive whitespaces case 2
					__a       :     List[str]								=	["""were   wolf"""]
					__a       :     Optional[int]								=	["""were               wolf"""]
					# S = 0, D = 0, I = 0, N = 9, CER = 0
					__a       :     Optional[Any]								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.0			) < 1E-6			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     Dict								=	["""werewolf"""]
					__a       :     str								=	["""weaewolf"""]
					# S = 1, D = 0, I = 0, N = 8, CER = 0.125
					__a       :     Optional[int]								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.1_2_5			) < 1E-6			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     List[str]								=	["""werewolf"""]
					__a       :     Tuple								=	["""wereawolf"""]
					# S = 0, D = 1, I = 0, N = 8, CER = 0.125
					__a       :     Optional[Any]								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.1_2_5			) < 1E-6			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     List[str]								=	["""werewolf"""]
					__a       :     Optional[Any]								=	["""wereolf"""]
					# S = 0, D = 0, I = 1, N = 8, CER = 0.125
					__a       :     int								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.1_2_5			) < 1E-6			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     Dict								=	["""werewolf"""]
					__a       :     Tuple								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertEqual(__UpperCamelCase  ,    0.0			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     List[str]								=	["""werewolf""", """I am your father"""]
					__a       :     int								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertEqual(__UpperCamelCase  ,    0.0			)

					__a       :     Dict								=	["""werewolf""", """I am your father""", """doge"""]
					__a       :     Optional[Any]								=	["""werxwolf""", """I       am your father""", """doge"""]
					# S = 1, D = 0, I = 0, N = 28, CER = 1 / 28
					__a       :     Union[str, Any]								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.0_3_5_7_1_4_2_8			) < 1E-6			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     List[str]								=	["""My hovercraft""", """is full of eels"""]
					__a       :     Union[str, Any]								=	["""My hovercraft is full""", """ of eels"""]
					# S = 0, D = 0, I = 2, N = 28, CER = 2 / 28
					# whitespace at the front of " of eels" will be strip during preporcessing
					# so need to insert 2 whitespaces
					__a       :     str								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase  ,    concatenate_texts=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.0_7_1_4_2_8			) < 1E-6			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     int								=	["""我能吞下玻璃而不伤身体"""]
					__a       :     Optional[Any]								=	[""" 能吞虾玻璃而 不霜身体啦"""]
					# S = 3, D = 2, I = 0, N = 11, CER = 5 / 11
					__a       :     List[Any]								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.4_5_4_5_4_5_4_5_4_5			) < 1E-6			)

					__a       :     Union[str, Any]								=	["""我能吞下玻璃""", """而不伤身体"""]
					__a       :     Optional[Any]								=	["""我    能 吞 下 玻 璃""", """而不伤身体"""]
					# S = 0, D = 5, I = 0, N = 11, CER = 5 / 11
					__a       :     List[str]								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.4_5_4_5_4_5_4_5_4_5_4_5			) < 1E-6			)

					__a       :     Union[str, Any]								=	["""我能吞下玻璃而不伤身体"""]
					__a       :     Dict								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertFalse(__UpperCamelCase  ,    0.0			)

	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     List[Any]								=	[""""""]
					__a       :     Dict								=	["""Hypothesis"""]
					with self.assertRaises(__UpperCamelCase			):
									cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)


if __name__ == "__main__":
					unittest.main()