'''simple docstring'''





import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class        __A      ( UpperCamelCase__		,	unittest.TestCase	):
    a__       :						List[Any]          =  GPTSanJapaneseTokenizer
    a__       :						Optional[Any]          =  False
    a__       :						List[str]          =  {"""do_clean_text""": False, """add_prefix_space""": False}



    def 					_lowercase						(self    :    Union[str, Any] ):
         super().setUp()

         # fmt: off
         UpperCAmelCase_           =   ["こん", "こんに", "にちは", "ばんは", "世界,㔺界", "、", "。", "<BR>", "<SP>", "<TAB>", "<URL>", "<EMAIL>", "<TEL>", "<DATE>", "<PRICE>", "<BLOCK>", "<KIGOU>", "<U2000U2BFF>", "<|emoji1|>", "<unk>", "<|bagoftoken|>", "<|endoftext|>"]
         # fmt: on
         UpperCAmelCase_           =   {"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
         UpperCAmelCase_           =   {"unk_token": "<unk>"}

         UpperCAmelCase_           =   os.path.join(self.tmpdirname   ,			VOCAB_FILES_NAMES["vocab_file"] )
         UpperCAmelCase_           =   os.path.join(self.tmpdirname   ,			VOCAB_FILES_NAMES["emoji_file"] )
         with open(self.vocab_file   ,			"w"   ,			encoding="utf-8" ) as vocab_writer:
              vocab_writer.write("".join([x + "\n" for x in vocab_tokens] ) )
         with open(self.emoji_file   ,			"w" ) as emoji_writer:
              emoji_writer.write(json.dumps(__a ) )



    def 					_lowercase						(self    :    List[Any]   ,			**__a    :    str ):
         kwargs.update(self.special_tokens_map )
         return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname   ,			**__a )



    def 					_lowercase						(self    :    str   ,			__a    :    Tuple ):
         UpperCAmelCase_           =   "こんにちは、世界。 \nこんばんは、㔺界。😀"
         UpperCAmelCase_           =   "こんにちは、世界。 \nこんばんは、世界。😀"
         return input_text, output_text



    def 					_lowercase						(self    :    str   ,			__a    :    List[Any] ):
         UpperCAmelCase_    , UpperCAmelCase_           =   self.get_input_output_texts(__a )
         UpperCAmelCase_           =   tokenizer.encode(__a   ,			add_special_tokens=__a )
         UpperCAmelCase_           =   tokenizer.decode(__a   ,			clean_up_tokenization_spaces=__a )
         return text, ids



    def 					_lowercase						(self    :    str ):
         pass  # TODO add if relevant



    def 					_lowercase						(self    :    str ):
         pass  # TODO add if relevant



    def 					_lowercase						(self    :    List[Any] ):
         pass  # TODO add if relevant



    def 					_lowercase						(self    :    Any ):
         UpperCAmelCase_           =   self.get_tokenizer()

         # Testing tokenization
         UpperCAmelCase_           =   "こんにちは、世界。　こんばんは、㔺界。"
         UpperCAmelCase_           =   ["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
         UpperCAmelCase_           =   tokenizer.tokenize(__a )
         self.assertListEqual(__a   ,			__a )

         # Testing conversion to ids without special tokens
         UpperCAmelCase_           =   [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
         UpperCAmelCase_           =   tokenizer.convert_tokens_to_ids(__a )
         self.assertListEqual(__a   ,			__a )

         # Testing conversion to ids with special tokens
         UpperCAmelCase_           =   tokens + [tokenizer.unk_token]
         UpperCAmelCase_           =   [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
         UpperCAmelCase_           =   tokenizer.convert_tokens_to_ids(__a )
         self.assertListEqual(__a   ,			__a )



    def 					_lowercase						(self    :    Tuple ):
         UpperCAmelCase_           =   self.get_tokenizer()

         # Testing tokenization
         UpperCAmelCase_           =   "こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"
         UpperCAmelCase_           =   "こんにちは、、、、世界。こんばんは、、、、世界。"
         UpperCAmelCase_           =   tokenizer.encode(__a )
         UpperCAmelCase_           =   tokenizer.decode(__a )
         self.assertEqual(__a   ,			__a )



    @slow
    def 					_lowercase						(self    :    Optional[Any] ):
         UpperCAmelCase_           =   self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese" )

         # Testing tokenization
         UpperCAmelCase_           =   "こんにちは、世界。"
         UpperCAmelCase_           =   "こんばんは、㔺界。😀"
         UpperCAmelCase_           =   "こんにちは、世界。こんばんは、世界。😀"
         UpperCAmelCase_           =   tokenizer.encode(prefix_text + input_text )
         UpperCAmelCase_           =   tokenizer.encode(""   ,			prefix_text=prefix_text + input_text )
         UpperCAmelCase_           =   tokenizer.encode(__a   ,			prefix_text=__a )
         UpperCAmelCase_           =   tokenizer.decode(__a )
         UpperCAmelCase_           =   tokenizer.decode(__a )
         UpperCAmelCase_           =   tokenizer.decode(__a )
         self.assertEqual(__a   ,			__a )
         self.assertEqual(__a   ,			__a )
         self.assertEqual(__a   ,			__a )



    @slow
    def 					_lowercase						(self    :    Optional[Any] ):
         UpperCAmelCase_           =   self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese" )

         # Testing tokenization
         UpperCAmelCase_           =   "こんにちは、世界。"
         UpperCAmelCase_           =   "こんばんは、㔺界。😀"

         UpperCAmelCase_           =   len(tokenizer.encode(__a ) ) - 2
         UpperCAmelCase_           =   len(tokenizer.encode(__a ) ) - 2

         UpperCAmelCase_           =   [1] + [0] * (len_prefix + len_text + 1)
         UpperCAmelCase_           =   [1] * (len_prefix + len_text + 1) + [0]
         UpperCAmelCase_           =   [1] + [1] * (len_prefix) + [0] * (len_text + 1)

         UpperCAmelCase_           =   tokenizer(prefix_text + input_text ).token_type_ids
         UpperCAmelCase_           =   tokenizer(""   ,			prefix_text=prefix_text + input_text ).token_type_ids
         UpperCAmelCase_           =   tokenizer(__a   ,			prefix_text=__a ).token_type_ids
         self.assertListEqual(__a   ,			__a )
         self.assertListEqual(__a   ,			__a )
         self.assertListEqual(__a   ,			__a )



    @slow
    def 					_lowercase						(self    :    Optional[int] ):
         UpperCAmelCase_           =   self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese" )

         UpperCAmelCase_           =   tokenizer.encode("あンいワ" )
         UpperCAmelCase_           =   tokenizer.encode(""   ,			prefix_text="あンいワ" )
         UpperCAmelCase_           =   tokenizer.encode("いワ"   ,			prefix_text="あン" )

         self.assertEqual(tokenizer.decode(__a )   ,			tokenizer.decode(__a ) )
         self.assertEqual(tokenizer.decode(__a )   ,			tokenizer.decode(__a ) )
         self.assertNotEqual(__a   ,			__a )
         self.assertNotEqual(__a   ,			__a )
         self.assertEqual(x_token_a[1]   ,			x_token_a[-1] )  # SEG token
         self.assertEqual(x_token_a[1]   ,			x_token_a[3] )  # SEG token



    @slow
    def 					_lowercase						(self    :    int ):
         UpperCAmelCase_           =   self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese" )

         UpperCAmelCase_           =   [["武田信玄", "は、"], ["織田信長", "の配下の、"]]
         UpperCAmelCase_           =   tokenizer(__a   ,			padding=__a )
         UpperCAmelCase_           =   tokenizer.batch_encode_plus(__a   ,			padding=__a )

         # fmt: off
         UpperCAmelCase_           =   [[35993, 8640, 25948, 35998, 30647, 35675, 35999, 35999], [35993, 10382, 9868, 35998, 30646, 9459, 30646, 35675]]
         UpperCAmelCase_           =   [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
         UpperCAmelCase_           =   [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
         # fmt: on
         self.assertListEqual(x_token.input_ids   ,			__a )
         self.assertListEqual(x_token.token_type_ids   ,			__a )
         self.assertListEqual(x_token.attention_mask   ,			__a )
         self.assertListEqual(x_token_a.input_ids   ,			__a )
         self.assertListEqual(x_token_a.token_type_ids   ,			__a )
         self.assertListEqual(x_token_a.attention_mask   ,			__a )



    def 					_lowercase						(self    :    List[Any] ):
         # Intentionally convert some words to accommodate character fluctuations unique to Japanese
         pass






    def 					_lowercase						(self    :    List[str] ):
         # tokenizer has no padding token
         pass





'''simple docstring'''





import math





def 	lowerCAmelCase_				(					)    ->      None:


     '''simple docstring'''




     UpperCAmelCase_           =   input("Enter message: "			)
     UpperCAmelCase_           =   int(input(f"""Enter key [2-{len(snake_case_			) - 1}]: """			)			)
     UpperCAmelCase_           =   input("Encryption/Decryption [e/d]: "			)

     if mode.lower().startswith("e"			):
          UpperCAmelCase_           =   encrypt_message(snake_case_					,    snake_case_			)
     elif mode.lower().startswith("d"			):
          UpperCAmelCase_           =   decrypt_message(snake_case_					,    snake_case_			)

     # Append pipe symbol (vertical bar) to identify spaces at the end.
     print(f"""Output:\n{text + "|"}"""			)





def 	lowerCAmelCase_				(					snake_case_       : int					,    snake_case_       : str			)    ->      str:


     '''simple docstring'''




     UpperCAmelCase_           =   [""] * key
     for col in range(snake_case_			):
          UpperCAmelCase_           =   col
          while pointer < len(snake_case_			):
               cipher_text[col] += message[pointer]
               pointer += key
     return "".join(snake_case_			)





def 	lowerCAmelCase_				(					snake_case_       : int					,    snake_case_       : str			)    ->      str:


     '''simple docstring'''




     UpperCAmelCase_           =   math.ceil(len(snake_case_			) / key			)
     UpperCAmelCase_           =   key
     UpperCAmelCase_           =   (num_cols * num_rows) - len(snake_case_			)
     UpperCAmelCase_           =   [""] * num_cols
     UpperCAmelCase_           =   0
     UpperCAmelCase_           =   0

     for symbol in message:
          plain_text[col] += symbol
          col += 1

          if (
              (col == num_cols)
              or (col == num_cols - 1)
              and (row >= num_rows - num_shaded_boxes)
          ):
               UpperCAmelCase_           =   0
               row += 1

     return "".join(snake_case_			)


if __name__ == "__main__":
    import doctest

    doctest.testmod()
    main()




