
import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 				UpperCamelCase__		(	lowerCamelCase__       ,    unittest.TestCase	):






       """simple docstring"""


       UpperCAmelCase_				       =GPTNeoXJapaneseTokenizer
       UpperCAmelCase_				       =False
       UpperCAmelCase_				       ={"do_clean_text": False, "add_prefix_space": False}



       def     _UpperCamelCase	(							self	)				->				Tuple:
          super().setUp()

          SCREAMING_SNAKE_CASE_	=      [
              '''こん''',
              '''こんに''',
              '''にちは''',
              '''ばんは''',
              '''世界,㔺界''',
              '''、''',
              '''。''',
              '''<BR>''',
              '''<SP>''',
              '''<TAB>''',
              '''<URL>''',
              '''<EMAIL>''',
              '''<TEL>''',
              '''<DATE>''',
              '''<PRICE>''',
              '''<BLOCK>''',
              '''<KIGOU>''',
              '''<U2000U2BFF>''',
              '''<|emoji1|>''',
              '''<unk>''',
              '''<|startoftext|>''',
              '''<|endoftext|>''',
          ]
          SCREAMING_SNAKE_CASE_	=      {'''emoji''': {'''\ud83d\ude00''': '''<|emoji1|>'''}, '''emoji_inv''': {'''<|emoji1|>''': '''\ud83d\ude00'''}}  # 😀
          SCREAMING_SNAKE_CASE_	=      {'''unk_token''': '''<unk>'''}

          SCREAMING_SNAKE_CASE_	=      os.path.join(self.tmpdirname ,     VOCAB_FILES_NAMES['''vocab_file''']	)
          SCREAMING_SNAKE_CASE_	=      os.path.join(self.tmpdirname ,     VOCAB_FILES_NAMES['''emoji_file''']	)
          with open(self.vocab_file ,     '''w''' ,     encoding='''utf-8'''	) as vocab_writer:
             vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]	)	)
          with open(self.emoji_file ,     '''w'''	) as emoji_writer:
             emoji_writer.write(json.dumps(__A	)	)



       def     _UpperCamelCase	(							self ,     **_A	)				->				List[Any]:
          kwargs.update(self.special_tokens_map	)
          return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname ,     **__A	)



       def     _UpperCamelCase	(							self ,     _A	)				->				str:
          SCREAMING_SNAKE_CASE_	=      '''こんにちは、世界。 \nこんばんは、㔺界。😀'''
          SCREAMING_SNAKE_CASE_	=      '''こんにちは、世界。 \nこんばんは、世界。😀'''
          return input_text, output_text



       def     _UpperCamelCase	(							self ,     _A	)				->				Any:
          SCREAMING_SNAKE_CASE_	=      self.get_input_output_texts(__A	)
          SCREAMING_SNAKE_CASE_	=      tokenizer.encode(__A ,     add_special_tokens=__A	)
          SCREAMING_SNAKE_CASE_	=      tokenizer.decode(__A ,     clean_up_tokenization_spaces=__A	)
          return text, ids



       def     _UpperCamelCase	(							self	)				->				Optional[int]:
          pass  # TODO add if relevant



       def     _UpperCamelCase	(							self	)				->				int:
          pass  # TODO add if relevant



       def     _UpperCamelCase	(							self	)				->				str:
          pass  # TODO add if relevant



       def     _UpperCamelCase	(							self	)				->				Dict:
          SCREAMING_SNAKE_CASE_	=      self.get_tokenizer()

          # Testing tokenization
          SCREAMING_SNAKE_CASE_	=      '''こんにちは、世界。　こんばんは、㔺界。'''
          SCREAMING_SNAKE_CASE_	=      ['''こん''', '''にちは''', '''、''', '''世界''', '''。''', '''<SP>''', '''こん''', '''ばんは''', '''、''', '''㔺界''', '''。''']
          SCREAMING_SNAKE_CASE_	=      tokenizer.tokenize(__A	)
          self.assertListEqual(__A ,     __A	)

          # Testing conversion to ids without special tokens
          SCREAMING_SNAKE_CASE_	=      [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
          SCREAMING_SNAKE_CASE_	=      tokenizer.convert_tokens_to_ids(__A	)
          self.assertListEqual(__A ,     __A	)

          # Testing conversion to ids with special tokens
          SCREAMING_SNAKE_CASE_	=      tokens + [tokenizer.unk_token]
          SCREAMING_SNAKE_CASE_	=      [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
          SCREAMING_SNAKE_CASE_	=      tokenizer.convert_tokens_to_ids(__A	)
          self.assertListEqual(__A ,     __A	)



       @slow
       def     _UpperCamelCase	(							self	)				->				int:
          SCREAMING_SNAKE_CASE_	=      self.tokenizer_class.from_pretrained('''abeja/gpt-neox-japanese-2.7b'''	)

          SCREAMING_SNAKE_CASE_	=      tokenizer.encode('''ありがとう。''' ,     add_special_tokens=__A	)
          SCREAMING_SNAKE_CASE_	=      tokenizer.encode('''どういたしまして。''' ,     add_special_tokens=__A	)

          SCREAMING_SNAKE_CASE_	=      tokenizer.build_inputs_with_special_tokens(__A	)
          SCREAMING_SNAKE_CASE_	=      tokenizer.build_inputs_with_special_tokens(__A ,     __A	)

          assert encoded_sentence == ids_a
          assert encoded_pair == ids_a + ids_a



       def     _UpperCamelCase	(							self	)				->				Optional[Any]:
          # Intentionally convert some words to accommodate character fluctuations unique to Japanese
          pass




       def     _UpperCamelCase	(							self	)				->				Union[str, Any]:
          # tokenizer has no padding token
          pass





def      a__	(						UpperCAmelCase				:     list     )    ->		list:
  UpperCAmelCase			:				Tuple										=					len(UpperCAmelCase     )

  # Set the gap value to be decreased by a factor of 1.3
  # after each iteration
  UpperCAmelCase			:				Optional[Any]										=					1.3

  # Continue sorting until the gap is 1
  while gap > 1:
    # Decrease the gap value
    UpperCAmelCase			:				int										=					int(gap / shrink     )

    # Sort the elements using insertion sort
    for i in range(UpperCAmelCase							,					len(UpperCAmelCase     )     ):
      UpperCAmelCase			:				Union[str, Any]										=					collection[i]
      UpperCAmelCase			:				List[Any]										=					i
      while j >= gap and collection[j - gap] > temp:
        UpperCAmelCase			:				int										=					collection[j - gap]
        j -= gap
      UpperCAmelCase			:				List[Any]										=					temp

  return collection


if __name__ == "__main__":
     import doctest

     doctest.testmod()





