'''simple docstring'''






import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


__a										=					get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
     from transformers.models.plbart.modeling_plbart import shift_tokens_right

__a										=					50_003
__a										=					50_002





@require_sentencepiece
@require_tokenizers
class     A__							(       UpperCamelCase__					,		unittest.TestCase   ):





   """simple docstring"""





   UpperCamelCase_       :						Union[str, Any]    				=     PLBartTokenizer
   UpperCamelCase_       :						Union[str, Any]    				=     None
   UpperCamelCase_       :						Tuple    				=     False




   def 			_lowerCAmelCase		( self     :		int			)       ->    List[str]:







      """simple docstring"""




      super().setUp()

      # We have a SentencePiece fixture for testing
      _UpperCAmelCase							:	Tuple							=   PLBartTokenizer(__a						,     language_codes="base"						,     keep_accents=__a			)
      tokenizer.save_pretrained(self.tmpdirname			)




   def 			_lowerCAmelCase		( self     :		List[str]			)       ->    List[Any]:







      """simple docstring"""




      _UpperCAmelCase							:	Optional[int]							=   PLBartTokenizer(__a						,     language_codes="base"						,     keep_accents=__a			)

      _UpperCAmelCase							:	Union[str, Any]							=   tokenizer.tokenize("This is a test"			)
      self.assertListEqual(__a						,     ["▁This", "▁is", "▁a", "▁t", "est"]			)

      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(__a			)						,     [value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]						,     )

      _UpperCAmelCase							:	List[str]							=   tokenizer.tokenize("I was born in 92000, and this is falsé."			)
      self.assertListEqual(
          __a						,     [
              SPIECE_UNDERLINE + "I",
              SPIECE_UNDERLINE + "was",
              SPIECE_UNDERLINE + "b",
              "or",
              "n",
              SPIECE_UNDERLINE + "in",
              SPIECE_UNDERLINE + "",
              "9",
              "2",
              "0",
              "0",
              "0",
              ",",
              SPIECE_UNDERLINE + "and",
              SPIECE_UNDERLINE + "this",
              SPIECE_UNDERLINE + "is",
              SPIECE_UNDERLINE + "f",
              "al",
              "s",
              "é",
              ".",
          ]						,     )
      _UpperCAmelCase							:	Tuple							=   tokenizer.convert_tokens_to_ids(__a			)
      self.assertListEqual(
          __a						,     [
              value + tokenizer.fairseq_offset
              for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
          ]						,     )

      _UpperCAmelCase							:	Dict							=   tokenizer.convert_ids_to_tokens(__a			)
      self.assertListEqual(
          __a						,     [
              SPIECE_UNDERLINE + "I",
              SPIECE_UNDERLINE + "was",
              SPIECE_UNDERLINE + "b",
              "or",
              "n",
              SPIECE_UNDERLINE + "in",
              SPIECE_UNDERLINE + "",
              "<unk>",
              "2",
              "0",
              "0",
              "0",
              ",",
              SPIECE_UNDERLINE + "and",
              SPIECE_UNDERLINE + "this",
              SPIECE_UNDERLINE + "is",
              SPIECE_UNDERLINE + "f",
              "al",
              "s",
              "<unk>",
              ".",
          ]						,     )

      _UpperCAmelCase							:	Any							=   tokenizer.vocab_size
      _UpperCAmelCase							:	Dict							=   [tokenizer.convert_ids_to_tokens(__a			) for x in range(end - 4						,     __a			)]

      self.assertListEqual(__a						,     ["__java__", "__python__", "__en_XX__", "<mask>"]			)

      _UpperCAmelCase							:	Tuple							=   "java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"
      _UpperCAmelCase							:	Any							=   tokenizer(__a			).input_ids
      self.assertEqual(
          tokenizer.decode(__a						,     skip_special_tokens=__a						,     clean_up_tokenization_spaces=__a			)						,     __a						,     )




   def 			_lowerCAmelCase		( self     :		List[str]			)       ->    Tuple:







      """simple docstring"""




      _UpperCAmelCase							:	str							=   PLBartTokenizer(__a						,     language_codes="multi"						,     keep_accents=__a			)

      _UpperCAmelCase							:	Optional[Any]							=   tokenizer.tokenize("This is a test"			)
      self.assertListEqual(__a						,     ["▁This", "▁is", "▁a", "▁t", "est"]			)

      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(__a			)						,     [value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]						,     )

      _UpperCAmelCase							:	str							=   tokenizer.tokenize("I was born in 92000, and this is falsé."			)
      self.assertListEqual(
          __a						,     [
              SPIECE_UNDERLINE + "I",
              SPIECE_UNDERLINE + "was",
              SPIECE_UNDERLINE + "b",
              "or",
              "n",
              SPIECE_UNDERLINE + "in",
              SPIECE_UNDERLINE + "",
              "9",
              "2",
              "0",
              "0",
              "0",
              ",",
              SPIECE_UNDERLINE + "and",
              SPIECE_UNDERLINE + "this",
              SPIECE_UNDERLINE + "is",
              SPIECE_UNDERLINE + "f",
              "al",
              "s",
              "é",
              ".",
          ]						,     )
      _UpperCAmelCase							:	Dict							=   tokenizer.convert_tokens_to_ids(__a			)
      self.assertListEqual(
          __a						,     [
              value + tokenizer.fairseq_offset
              for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
          ]						,     )

      _UpperCAmelCase							:	Optional[Any]							=   tokenizer.convert_ids_to_tokens(__a			)
      self.assertListEqual(
          __a						,     [
              SPIECE_UNDERLINE + "I",
              SPIECE_UNDERLINE + "was",
              SPIECE_UNDERLINE + "b",
              "or",
              "n",
              SPIECE_UNDERLINE + "in",
              SPIECE_UNDERLINE + "",
              "<unk>",
              "2",
              "0",
              "0",
              "0",
              ",",
              SPIECE_UNDERLINE + "and",
              SPIECE_UNDERLINE + "this",
              SPIECE_UNDERLINE + "is",
              SPIECE_UNDERLINE + "f",
              "al",
              "s",
              "<unk>",
              ".",
          ]						,     )
      _UpperCAmelCase							:	List[Any]							=   tokenizer.vocab_size
      _UpperCAmelCase							:	List[Any]							=   [tokenizer.convert_ids_to_tokens(__a			) for x in range(end - 7						,     __a			)]

      self.assertListEqual(
          __a						,     ["__java__", "__python__", "__en_XX__", "__javascript__", "__php__", "__ruby__", "__go__"]			)
      _UpperCAmelCase							:	Optional[Any]							=   "java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"
      _UpperCAmelCase							:	Dict							=   tokenizer(__a			).input_ids
      self.assertEqual(
          tokenizer.decode(__a						,     skip_special_tokens=__a						,     clean_up_tokenization_spaces=__a			)						,     __a						,     )





@require_torch
@require_sentencepiece
@require_tokenizers
class     A__							(       unittest.TestCase   ):





   """simple docstring"""





   UpperCamelCase_       :						str    				=     """uclanlp/plbart-python-en_XX"""
   UpperCamelCase_       :						List[str]    				=     [
       """def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])""",
       """def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])""",
   ]
   UpperCamelCase_       :						Tuple    				=     [
       """Returns the maximum value of a b c.""",
       """Sums the values of a b c.""",
   ]
   UpperCamelCase_       :						Optional[int]    				=     [
       1_34,
       54_52,
       3_34_60,
       3_34_41,
       3_34_63,
       3_34_65,
       3_34_63,
       3_34_49,
       9_88,
       20,
       3_34_56,
       19,
       3_34_56,
       7_71,
       39,
       42_58,
       8_89,
       33_18,
       3_34_41,
       3_34_63,
       3_34_65,
       3_34_63,
       3_34_49,
       24_71,
       2,
       PYTHON_CODE,
   ]




   @classmethod
   def 			_lowerCAmelCase		( cls     :		Optional[int]			)       ->    Dict:







      """simple docstring"""




      _UpperCAmelCase							:	int							=   PLBartTokenizer.from_pretrained(
          cls.checkpoint_name						,     language_codes="base"						,     src_lang="python"						,     tgt_lang="en_XX"			)
      _UpperCAmelCase							:	List[str]							=   1
      return cls




   def 			_lowerCAmelCase		( self     :		Union[str, Any]			)       ->    Any:







      """simple docstring"""




      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__java__"]						,     5_0_0_0_1			)
      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__python__"]						,     5_0_0_0_2			)
      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__en_XX__"]						,     5_0_0_0_3			)




   def 			_lowerCAmelCase		( self     :		Tuple			)       ->    str:







      """simple docstring"""




      _UpperCAmelCase							:	int							=   self.tokenizer.batch_encode_plus(self.src_text			).input_ids[0]
      self.assertListEqual(self.expected_src_tokens						,     __a			)




   def 			_lowerCAmelCase		( self     :		Dict			)       ->    List[str]:







      """simple docstring"""




      self.assertIn(__a						,     self.tokenizer.all_special_ids			)
      _UpperCAmelCase							:	List[Any]							=   [EN_CODE, 9_0_3_7, 3_3_4_4_2, 5_7, 7_5_2, 1_5_3, 1_4, 5_6, 1_8, 9, 2]
      _UpperCAmelCase							:	Tuple							=   self.tokenizer.decode(__a						,     skip_special_tokens=__a			)
      _UpperCAmelCase							:	List[str]							=   self.tokenizer.decode(generated_ids[1:]						,     skip_special_tokens=__a			)
      self.assertEqual(__a						,     __a			)
      self.assertNotIn(self.tokenizer.eos_token						,     __a			)




   def 			_lowerCAmelCase		( self     :		int			)       ->    Dict:







      """simple docstring"""




      _UpperCAmelCase							:	List[str]							=   ["def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])" * 2_0]
      self.assertIsInstance(src_text[0]						,     __a			)
      _UpperCAmelCase							:	int							=   1_0
      _UpperCAmelCase							:	int							=   self.tokenizer(__a						,     max_length=__a						,     truncation=__a			).input_ids[0]
      self.assertEqual(ids[-2]						,     2			)
      self.assertEqual(ids[-1]						,     __a			)
      self.assertEqual(len(__a			)						,     __a			)




   def 			_lowerCAmelCase		( self     :		str			)       ->    List[str]:







      """simple docstring"""




      self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "__java__"]			)						,     [5_0_0_0_4, 5_0_0_0_1]			)




   def 			_lowerCAmelCase		( self     :		int			)       ->    str:







      """simple docstring"""




      _UpperCAmelCase							:	List[Any]							=   tempfile.mkdtemp()
      _UpperCAmelCase							:	Tuple							=   self.tokenizer.fairseq_tokens_to_ids
      self.tokenizer.save_pretrained(__a			)
      _UpperCAmelCase							:	List[str]							=   PLBartTokenizer.from_pretrained(__a			)
      self.assertDictEqual(new_tok.fairseq_tokens_to_ids						,     __a			)




   @require_torch
   def 			_lowerCAmelCase		( self     :		str			)       ->    Dict:







      """simple docstring"""




      _UpperCAmelCase							:	Dict							=   self.tokenizer(self.src_text						,     text_target=self.tgt_text						,     padding=__a						,     return_tensors="pt"			)
      _UpperCAmelCase							:	str							=   shift_tokens_right(batch["labels"]						,     self.tokenizer.pad_token_id			)

      # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
      self.assertEqual(batch.input_ids[1][-2:].tolist()						,     [2, PYTHON_CODE]			)
      self.assertEqual(batch.decoder_input_ids[1][0]						,     __a			)
      self.assertEqual(batch.decoder_input_ids[1][-1]						,     2			)
      self.assertEqual(batch.labels[1][-2:].tolist()						,     [2, EN_CODE]			)




   @require_torch
   def 			_lowerCAmelCase		( self     :		Optional[Any]			)       ->    Union[str, Any]:







      """simple docstring"""




      _UpperCAmelCase							:	List[Any]							=   self.tokenizer(
          self.src_text						,     text_target=self.tgt_text						,     padding=__a						,     truncation=__a						,     max_length=len(self.expected_src_tokens			)						,     return_tensors="pt"						,     )
      _UpperCAmelCase							:	List[Any]							=   shift_tokens_right(batch["labels"]						,     self.tokenizer.pad_token_id			)

      self.assertIsInstance(__a						,     __a			)

      self.assertEqual((2, 2_6)						,     batch.input_ids.shape			)
      self.assertEqual((2, 2_6)						,     batch.attention_mask.shape			)
      _UpperCAmelCase							:	List[str]							=   batch.input_ids.tolist()[0]
      self.assertListEqual(self.expected_src_tokens						,     __a			)
      self.assertEqual(2						,     batch.decoder_input_ids[0, -1]			)  # EOS
      # Test that special tokens are reset
      self.assertEqual(self.tokenizer.prefix_tokens						,     []			)
      self.assertEqual(self.tokenizer.suffix_tokens						,     [self.tokenizer.eos_token_id, PYTHON_CODE]			)




   def 			_lowerCAmelCase		( self     :		Optional[int]			)       ->    List[Any]:







      """simple docstring"""




      _UpperCAmelCase							:	Optional[Any]							=   self.tokenizer(self.src_text						,     padding=__a						,     truncation=__a						,     max_length=3						,     return_tensors="pt"			)
      _UpperCAmelCase							:	List[str]							=   self.tokenizer(
          text_target=self.tgt_text						,     padding=__a						,     truncation=__a						,     max_length=1_0						,     return_tensors="pt"			)
      _UpperCAmelCase							:	Optional[int]							=   targets["input_ids"]
      _UpperCAmelCase							:	Dict							=   shift_tokens_right(__a						,     self.tokenizer.pad_token_id			)

      self.assertEqual(batch.input_ids.shape[1]						,     3			)
      self.assertEqual(batch.decoder_input_ids.shape[1]						,     1_0			)




   @require_torch
   def 			_lowerCAmelCase		( self     :		List[Any]			)       ->    Dict:







      """simple docstring"""




      _UpperCAmelCase							:	int							=   self.tokenizer._build_translation_inputs(
          "A test"						,     return_tensors="pt"						,     src_lang="en_XX"						,     tgt_lang="java"			)

      self.assertEqual(
          nested_simplify(__a			)						,     {
              # A, test, EOS, en_XX
              "input_ids": [[1_5_0, 2_4_2, 2, 5_0_0_0_3]],
              "attention_mask": [[1, 1, 1, 1]],
              # java
              "forced_bos_token_id": 5_0_0_0_1,
          }						,     )
'''simple docstring'''





import math





def 	lowerCAmelCase_				(					)    ->      None:


     '''simple docstring'''




     UpperCAmelCase_           =   input("Enter message: "			)
     UpperCAmelCase_           =   int(input(f"""Enter key [2-{len(snake_case_			) - 1}]: """			)			)
     UpperCAmelCase_           =   input("Encryption/Decryption [e/d]: "			)

     if mode.lower().startswith("e"			):
          UpperCAmelCase_           =   encrypt_message(snake_case_					,    snake_case_			)
     elif mode.lower().startswith("d"			):
          UpperCAmelCase_           =   decrypt_message(snake_case_					,    snake_case_			)

     # Append pipe symbol (vertical bar) to identify spaces at the end.
     print(f"""Output:\n{text + "|"}"""			)





def 	lowerCAmelCase_				(					snake_case_       : int					,    snake_case_       : str			)    ->      str:


     '''simple docstring'''




     UpperCAmelCase_           =   [""] * key
     for col in range(snake_case_			):
          UpperCAmelCase_           =   col
          while pointer < len(snake_case_			):
               cipher_text[col] += message[pointer]
               pointer += key
     return "".join(snake_case_			)





def 	lowerCAmelCase_				(					snake_case_       : int					,    snake_case_       : str			)    ->      str:


     '''simple docstring'''




     UpperCAmelCase_           =   math.ceil(len(snake_case_			) / key			)
     UpperCAmelCase_           =   key
     UpperCAmelCase_           =   (num_cols * num_rows) - len(snake_case_			)
     UpperCAmelCase_           =   [""] * num_cols
     UpperCAmelCase_           =   0
     UpperCAmelCase_           =   0

     for symbol in message:
          plain_text[col] += symbol
          col += 1

          if (
              (col == num_cols)
              or (col == num_cols - 1)
              and (row >= num_rows - num_shaded_boxes)
          ):
               UpperCAmelCase_           =   0
               row += 1

     return "".join(snake_case_			)


if __name__ == "__main__":
    import doctest

    doctest.testmod()
    main()




