


"""simple docstring"""







import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class       __A				(       __a,    unittest.TestCase			):






 """simple docstring"""




 __lowerCAmelCase 					=  CodeGenTokenizer
 __lowerCAmelCase 					=  CodeGenTokenizerFast
 __lowerCAmelCase 					=  True
 __lowerCAmelCase 					=  {"add_prefix_space": True}
 __lowerCAmelCase 					=  False


 def       SCREAMING_SNAKE_CASE							(		self						)					->     str:
     super().setUp()

     # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
     a 		=[
         """l""",
         """o""",
         """w""",
         """e""",
         """r""",
         """s""",
         """t""",
         """i""",
         """d""",
         """n""",
         """\u0120""",
         """\u0120l""",
         """\u0120n""",
         """\u0120lo""",
         """\u0120low""",
         """er""",
         """\u0120lowest""",
         """\u0120newer""",
         """\u0120wider""",
         """<unk>""",
         """<|endoftext|>""",
     ]
     a 		=dict(zip(a_ ,    range(len(a_						)						)						)						)
     a 		=["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
     a 		={"""unk_token""": """<unk>"""}

     a 		=os.path.join(self.tmpdirname ,    VOCAB_FILES_NAMES['''vocab_file''']						)
     a 		=os.path.join(self.tmpdirname ,    VOCAB_FILES_NAMES['''merges_file''']						)
     with open(self.vocab_file ,    '''w''' ,    encoding='''utf-8'''						) as fp:
         fp.write(json.dumps(a_						) + '''\n'''						)
     with open(self.merges_file ,    '''w''' ,    encoding='''utf-8'''						) as fp:
         fp.write('''\n'''.join(a_						)						)


 def       SCREAMING_SNAKE_CASE							(		self ,    **__A						)					->     Tuple:
     kwargs.update(self.special_tokens_map						)
     return CodeGenTokenizer.from_pretrained(self.tmpdirname ,    **a_						)


 def       SCREAMING_SNAKE_CASE							(		self ,    **__A						)					->     int:
     kwargs.update(self.special_tokens_map						)
     return CodeGenTokenizerFast.from_pretrained(self.tmpdirname ,    **a_						)


 def       SCREAMING_SNAKE_CASE							(		self ,    __A						)					->     Dict:
     a 		="""lower newer"""
     a 		="""lower newer"""
     return input_text, output_text


 def       SCREAMING_SNAKE_CASE							(		self						)					->     Any:
     a 		=CodeGenTokenizer(self.vocab_file ,    self.merges_file ,    **self.special_tokens_map						)
     a 		="""lower newer"""
     a 		=["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
     a 		=tokenizer.tokenize(a_ ,    add_prefix_space=a_						)
     self.assertListEqual(a_ ,    a_						)

     a 		=tokens + [tokenizer.unk_token]
     a 		=[14, 15, 10, 9, 3, 2, 15, 19]
     self.assertListEqual(tokenizer.convert_tokens_to_ids(a_						) ,    a_						)


 def       SCREAMING_SNAKE_CASE							(		self						)					->     Optional[int]:



     if not self.test_rust_tokenizer:
         return

     a 		=self.get_tokenizer()
     a 		=self.get_rust_tokenizer(add_prefix_space=a_						)

     a 		="""lower newer"""

     # Testing tokenization
     a 		=tokenizer.tokenize(a_ ,    add_prefix_space=a_						)
     a 		=rust_tokenizer.tokenize(a_						)
     self.assertListEqual(a_ ,    a_						)

     # Testing conversion to ids without special tokens
     a 		=tokenizer.encode(a_ ,    add_special_tokens=a_ ,    add_prefix_space=a_						)
     a 		=rust_tokenizer.encode(a_ ,    add_special_tokens=a_						)
     self.assertListEqual(a_ ,    a_						)

     # Testing conversion to ids with special tokens
     a 		=self.get_rust_tokenizer(add_prefix_space=a_						)
     a 		=tokenizer.encode(a_ ,    add_prefix_space=a_						)
     a 		=rust_tokenizer.encode(a_						)
     self.assertListEqual(a_ ,    a_						)

     # Testing the unknown token
     a 		=tokens + [rust_tokenizer.unk_token]
     a 		=[14, 15, 10, 9, 3, 2, 15, 19]
     self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(a_						) ,    a_						)


 def       SCREAMING_SNAKE_CASE							(		self ,    *__A ,    **__A						)					->     Union[str, Any]:
     pass


 def       SCREAMING_SNAKE_CASE							(		self ,    __A=15						)					->     Optional[int]:



     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
             a 		=self.rust_tokenizer_class.from_pretrained(a_ ,    **a_						)

             # Simple input
             a 		="""This is a simple input"""
             a 		=["""This is a simple input 1""", """This is a simple input 2"""]
             a 		=("""This is a simple input""", """This is a pair""")
             a 		=[
                 ("""This is a simple input 1""", """This is a simple input 2"""),
                 ("""This is a simple pair 1""", """This is a simple pair 2"""),
             ]

             # Simple input tests
             self.assertRaises(a_ ,    tokenizer_r.encode ,    a_ ,    max_length=a_ ,    padding='''max_length'''						)

             # Simple input
             self.assertRaises(a_ ,    tokenizer_r.encode_plus ,    a_ ,    max_length=a_ ,    padding='''max_length'''						)

             # Simple input
             self.assertRaises(
                 a_ ,    tokenizer_r.batch_encode_plus ,    a_ ,    max_length=a_ ,    padding='''max_length''' ,    )

             # Pair input
             self.assertRaises(a_ ,    tokenizer_r.encode ,    a_ ,    max_length=a_ ,    padding='''max_length'''						)

             # Pair input
             self.assertRaises(a_ ,    tokenizer_r.encode_plus ,    a_ ,    max_length=a_ ,    padding='''max_length'''						)

             # Pair input
             self.assertRaises(
                 a_ ,    tokenizer_r.batch_encode_plus ,    a_ ,    max_length=a_ ,    padding='''max_length''' ,    )


 def       SCREAMING_SNAKE_CASE							(		self						)					->     Dict:
     a 		=CodeGenTokenizer.from_pretrained(self.tmpdirname ,    pad_token='''<pad>'''						)

     # Simple input
     a 		="""This is a simple input"""
     a 		=["""This is a simple input looooooooong""", """This is a simple input"""]
     a 		=("""This is a simple input""", """This is a pair""")
     a 		=[
         ("""This is a simple input loooooong""", """This is a simple input"""),
         ("""This is a simple pair loooooong""", """This is a simple pair"""),
     ]

     a 		=tokenizer.pad_token_id

     a 		=tokenizer(a_ ,    padding='''max_length''' ,    max_length=30 ,    return_tensors='''np'''						)
     a 		=tokenizer(a_ ,    padding=a_ ,    truncate=a_ ,    return_tensors='''np'''						)
     a 		=tokenizer(*a_ ,    padding='''max_length''' ,    max_length=60 ,    return_tensors='''np'''						)
     a 		=tokenizer(a_ ,    padding=a_ ,    truncate=a_ ,    return_tensors='''np'''						)

     # s
     # test single string max_length padding
     self.assertEqual(out_s['''input_ids'''].shape[-1] ,    30						)
     self.assertTrue(pad_token_id in out_s['''input_ids''']						)
     self.assertTrue(0 in out_s['''attention_mask''']						)

     # s2
     # test automatic padding
     self.assertEqual(out_sa['''input_ids'''].shape[-1] ,    33						)
     # long slice doesn't have padding
     self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]						)
     self.assertFalse(0 in out_sa['''attention_mask'''][0]						)
     # short slice does have padding
     self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]						)
     self.assertTrue(0 in out_sa['''attention_mask'''][1]						)

     # p
     # test single pair max_length padding
     self.assertEqual(out_p['''input_ids'''].shape[-1] ,    60						)
     self.assertTrue(pad_token_id in out_p['''input_ids''']						)
     self.assertTrue(0 in out_p['''attention_mask''']						)

     # p2
     # test automatic padding pair
     self.assertEqual(out_pa['''input_ids'''].shape[-1] ,    52						)
     # long slice pair doesn't have padding
     self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]						)
     self.assertFalse(0 in out_pa['''attention_mask'''][0]						)
     # short slice pair does have padding
     self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]						)
     self.assertTrue(0 in out_pa['''attention_mask'''][1]						)


 def       SCREAMING_SNAKE_CASE							(		self						)					->     int:
     a 		="""$$$"""
     a 		=CodeGenTokenizer.from_pretrained(self.tmpdirname ,    bos_token=a_ ,    add_bos_token=a_						)

     a 		="""This is a simple input"""
     a 		=["""This is a simple input 1""", """This is a simple input 2"""]

     a 		=tokenizer.bos_token_id

     a 		=tokenizer(a_						)
     a 		=tokenizer(a_						)

     self.assertEqual(out_s.input_ids[0] ,    a_						)
     self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids						)						)

     a 		=tokenizer.decode(out_s.input_ids						)
     a 		=tokenizer.batch_decode(out_sa.input_ids						)

     self.assertEqual(decode_s.split()[0] ,    a_						)
     self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa						)						)


 @slow
 def       SCREAMING_SNAKE_CASE							(		self						)					->     Optional[Any]:
     a 		=CodeGenTokenizer.from_pretrained('''Salesforce/codegen-350M-mono'''						)

     a 		="""\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
     a 		="""\nif len_a > len_b:      result = a\nelse:      result = b"""

     a 		=tokenizer.encode(a_						)
     a 		=["""^#""", re.escape('''<|endoftext|>'''						), """^'''""", """^\"\"\"""", """\n\n\n"""]
     a 		=tokenizer.decode(a_ ,    truncate_before_pattern=a_						)
     self.assertEqual(a_ ,    a_						)





 def       SCREAMING_SNAKE_CASE							(		self						)					->     str:
     pass





"""simple docstring"""


import argparse
import hashlib  # hashlib is only used inside the Test class
import struct






class 					lowercase:







 '''simple docstring'''





 def __init__(				self:		List[Any],      a_:		List[str]		):




   '''simple docstring'''



   _snake_case							:		int								=  data
   _snake_case							:		Dict								=  [0X67452301, 0Xefcdab89, 0X98badcfe, 0X10325476, 0Xc3d2e1f0]


 @staticmethod
 def  UpperCamelCase_   (				a_:		Optional[Any],      a_:		Dict		):




   '''simple docstring'''



   return ((n << b) | (n >> (32 - b))) & 0Xffffffff


 def  UpperCamelCase_   (				self:		List[Any]		):




   '''simple docstring'''



   _snake_case							:		Union[str, Any]								=  B"""\x80""" + B"""\x00""" * (63 - (len(self.data		) + 8) % 64)
   _snake_case							:		Optional[int]								=  self.data + padding + struct.pack(""">Q""",      8 * len(self.data		)		)
   return padded_data


 def  UpperCamelCase_   (				self:		Union[str, Any]		):




   '''simple docstring'''



   return [
       self.padded_data[i : i + 64] for i in range(0,      len(self.padded_data		),      64		)
   ]


 def  UpperCamelCase_   (				self:		Optional[Any],      a_:		List[Any]		):




   '''simple docstring'''



   _snake_case							:		List[str]								=  list(struct.unpack(""">16L""",      a_		)		) + [0] * 64
   for i in range(16,      80		):
     _snake_case							:		List[Any]								=  self.rotate((w[i - 3] ^ w[i - 8] ^ w[i - 14] ^ w[i - 16]),      1		)
   return w


 def  UpperCamelCase_   (				self:		int		):




   '''simple docstring'''



   _snake_case							:		Union[str, Any]								=  self.padding()
   _snake_case							:		str								=  self.split_blocks()
   for block in self.blocks:
     _snake_case							:		Any								=  self.expand_block(a_		)
     _snake_case		,					_snake_case		,					_snake_case		,					_snake_case		,					_snake_case							:		Optional[int]								=  self.h
     for i in range(0,      80		):
       if 0 <= i < 20:
         _snake_case							:		int								=  (b & c) | ((~b) & d)
         _snake_case							:		str								=  0X5a827999
       elif 20 <= i < 40:
         _snake_case							:		Optional[int]								=  b ^ c ^ d
         _snake_case							:		str								=  0X6ed9eba1
       elif 40 <= i < 60:
         _snake_case							:		List[Any]								=  (b & c) | (b & d) | (c & d)
         _snake_case							:		List[Any]								=  0X8f1bbcdc
       elif 60 <= i < 80:
         _snake_case							:		List[Any]								=  b ^ c ^ d
         _snake_case							:		int								=  0Xca62c1d6
       _snake_case		,					_snake_case		,					_snake_case		,					_snake_case		,					_snake_case							:		Optional[int]								=  (
           self.rotate(a_,      5		) + f + e + k + expanded_block[i] & 0Xffffffff,
           a,
           self.rotate(a_,      30		),
           c,
           d,
       )
     _snake_case							:		Union[str, Any]								=  (
         self.h[0] + a & 0Xffffffff,
         self.h[1] + b & 0Xffffffff,
         self.h[2] + c & 0Xffffffff,
         self.h[3] + d & 0Xffffffff,
         self.h[4] + e & 0Xffffffff,
     )
   return ("{:08x}" * 5).format(*self.h		)







def 					UpperCAmelCase__       ():
  """simple docstring"""

  _snake_case							:		Any								=  B"""Test String"""
  assert SHAaHash(snake_case__    ).final_hash() == hashlib.shaa(snake_case__    ).hexdigest()  # noqa: S324


def 					UpperCAmelCase__       ():
  """simple docstring"""

  _snake_case							:		List[Any]								=  argparse.ArgumentParser(description="""Process some strings or files"""    )
  parser.add_argument(
      """--string"""		,   dest="""input_string"""		,   default="""Hello World!! Welcome to Cryptography"""		,   help="""Hash the string"""		,   )
  parser.add_argument("""--file"""		,   dest="""input_file"""		,   help="""Hash contents of a file"""    )
  _snake_case							:		Union[str, Any]								=  parser.parse_args()
  _snake_case							:		List[Any]								=  args.input_string
  # In any case hash input should be a bytestring
  if args.input_file:
    with open(args.input_file		,   """rb"""    ) as f:
      _snake_case							:		str								=  f.read()
  else:
    _snake_case							:		int								=  bytes(snake_case__		,   """utf-8"""    )
  print(SHAaHash(snake_case__    ).final_hash()    )


if __name__ == "__main__":
     main()
     import doctest

     doctest.testmod()


