import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 	__snake_case      (   A__    ,      unittest.TestCase      ):
 __lowerCamelCase			:      List[Any]           =					CodeGenTokenizer
 __lowerCamelCase			:      str           =					CodeGenTokenizerFast
 __lowerCamelCase			:      str           =					True
 __lowerCamelCase			:      Optional[int]           =					{"""add_prefix_space""": True}
 __lowerCamelCase			:      Union[str, Any]           =					False
 def 		UpperCAmelCase__				(     self  )		->							Any:
       '''simple docstring'''





       super().setUp()

       # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
       UpperCAmelCase			:       Optional[int]			     =[
           "l",
           "o",
           "w",
           "e",
           "r",
           "s",
           "t",
           "i",
           "d",
           "n",
           "\u0120",
           "\u0120l",
           "\u0120n",
           "\u0120lo",
           "\u0120low",
           "er",
           "\u0120lowest",
           "\u0120newer",
           "\u0120wider",
           "<unk>",
           "<|endoftext|>",
       ]
       UpperCAmelCase			:       Optional[int]			     =dict(zip(_A						,      range(len(_A  )  )  )  )
       UpperCAmelCase			:       Union[str, Any]			     =["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
       UpperCAmelCase			:       List[str]			     ={"unk_token": "<unk>"}

       UpperCAmelCase			:       Tuple			     =os.path.join(self.tmpdirname						,      VOCAB_FILES_NAMES['''vocab_file''']  )
       UpperCAmelCase			:       Any			     =os.path.join(self.tmpdirname						,      VOCAB_FILES_NAMES['''merges_file''']  )
       with open(self.vocab_file						,      '''w'''						,      encoding='''utf-8'''  ) as fp:
             fp.write(json.dumps(_A  ) + '''\n'''  )
       with open(self.merges_file						,      '''w'''						,      encoding='''utf-8'''  ) as fp:
             fp.write('''\n'''.join(_A  )  )
 def 		UpperCAmelCase__				(     self						,      **snake_case__  )		->							Dict:
       '''simple docstring'''





       kwargs.update(self.special_tokens_map  )
       return CodeGenTokenizer.from_pretrained(self.tmpdirname						,      **_A  )
 def 		UpperCAmelCase__				(     self						,      **snake_case__  )		->							Union[str, Any]:
       '''simple docstring'''





       kwargs.update(self.special_tokens_map  )
       return CodeGenTokenizerFast.from_pretrained(self.tmpdirname						,      **_A  )
 def 		UpperCAmelCase__				(     self						,      snake_case__  )		->							Optional[Any]:
       '''simple docstring'''





       UpperCAmelCase			:       Any			     ="lower newer"
       UpperCAmelCase			:       Dict			     ="lower newer"
       return input_text, output_text
 def 		UpperCAmelCase__				(     self  )		->							Any:
       '''simple docstring'''





       UpperCAmelCase			:       List[str]			     =CodeGenTokenizer(self.vocab_file						,      self.merges_file						,      **self.special_tokens_map  )
       UpperCAmelCase			:       int			     ="lower newer"
       UpperCAmelCase			:       List[Any]			     =["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
       UpperCAmelCase			:       Tuple			     =tokenizer.tokenize(_A						,      add_prefix_space=_A  )
       self.assertListEqual(_A						,      _A  )

       UpperCAmelCase			:       int			     =tokens + [tokenizer.unk_token]
       UpperCAmelCase			:       List[str]			     =[14, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_A  )						,      _A  )
 def 		UpperCAmelCase__				(     self  )		->							Optional[int]:
       '''simple docstring'''





       if not self.test_rust_tokenizer:
             return

       UpperCAmelCase			:       Tuple			     =self.get_tokenizer()
       UpperCAmelCase			:       Any			     =self.get_rust_tokenizer(add_prefix_space=_A  )

       UpperCAmelCase			:       List[str]			     ="lower newer"

       # Testing tokenization
       UpperCAmelCase			:       Optional[Any]			     =tokenizer.tokenize(_A						,      add_prefix_space=_A  )
       UpperCAmelCase			:       Any			     =rust_tokenizer.tokenize(_A  )
       self.assertListEqual(_A						,      _A  )

       # Testing conversion to ids without special tokens
       UpperCAmelCase			:       int			     =tokenizer.encode(_A						,      add_special_tokens=_A						,      add_prefix_space=_A  )
       UpperCAmelCase			:       Optional[Any]			     =rust_tokenizer.encode(_A						,      add_special_tokens=_A  )
       self.assertListEqual(_A						,      _A  )

       # Testing conversion to ids with special tokens
       UpperCAmelCase			:       int			     =self.get_rust_tokenizer(add_prefix_space=_A  )
       UpperCAmelCase			:       int			     =tokenizer.encode(_A						,      add_prefix_space=_A  )
       UpperCAmelCase			:       Optional[int]			     =rust_tokenizer.encode(_A  )
       self.assertListEqual(_A						,      _A  )

       # Testing the unknown token
       UpperCAmelCase			:       int			     =tokens + [rust_tokenizer.unk_token]
       UpperCAmelCase			:       Any			     =[14, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(_A  )						,      _A  )
 def 		UpperCAmelCase__				(     self						,      *snake_case__						,      **snake_case__  )		->							str:
       '''simple docstring'''





       pass
 def 		UpperCAmelCase__				(     self						,      snake_case__=15  )		->							str:
       '''simple docstring'''





       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
                   UpperCAmelCase			:       Tuple			     =self.rust_tokenizer_class.from_pretrained(_A						,      **_A  )

                   # Simple input
                   UpperCAmelCase			:       Optional[int]			     ="This is a simple input"
                   UpperCAmelCase			:       Dict			     =["This is a simple input 1", "This is a simple input 2"]
                   UpperCAmelCase			:       Optional[Any]			     =("This is a simple input", "This is a pair")
                   UpperCAmelCase			:       Union[str, Any]			     =[
                       ("This is a simple input 1", "This is a simple input 2"),
                       ("This is a simple pair 1", "This is a simple pair 2"),
                   ]

                   # Simple input tests
                   self.assertRaises(_A						,      tokenizer_r.encode						,      _A						,      max_length=_A						,      padding='''max_length'''  )

                   # Simple input
                   self.assertRaises(_A						,      tokenizer_r.encode_plus						,      _A						,      max_length=_A						,      padding='''max_length'''  )

                   # Simple input
                   self.assertRaises(
                       _A						,      tokenizer_r.batch_encode_plus						,      _A						,      max_length=_A						,      padding='''max_length'''						,      )

                   # Pair input
                   self.assertRaises(_A						,      tokenizer_r.encode						,      _A						,      max_length=_A						,      padding='''max_length'''  )

                   # Pair input
                   self.assertRaises(_A						,      tokenizer_r.encode_plus						,      _A						,      max_length=_A						,      padding='''max_length'''  )

                   # Pair input
                   self.assertRaises(
                       _A						,      tokenizer_r.batch_encode_plus						,      _A						,      max_length=_A						,      padding='''max_length'''						,      )
 def 		UpperCAmelCase__				(     self  )		->							List[Any]:
       '''simple docstring'''





       UpperCAmelCase			:       int			     =CodeGenTokenizer.from_pretrained(self.tmpdirname						,      pad_token='''<pad>'''  )

       # Simple input
       UpperCAmelCase			:       Any			     ="This is a simple input"
       UpperCAmelCase			:       str			     =["This is a simple input looooooooong", "This is a simple input"]
       UpperCAmelCase			:       int			     =("This is a simple input", "This is a pair")
       UpperCAmelCase			:       List[str]			     =[
           ("This is a simple input loooooong", "This is a simple input"),
           ("This is a simple pair loooooong", "This is a simple pair"),
       ]

       UpperCAmelCase			:       Dict			     =tokenizer.pad_token_id

       UpperCAmelCase			:       str			     =tokenizer(_A						,      padding='''max_length'''						,      max_length=30						,      return_tensors='''np'''  )
       UpperCAmelCase			:       Tuple			     =tokenizer(_A						,      padding=_A						,      truncate=_A						,      return_tensors='''np'''  )
       UpperCAmelCase			:       Union[str, Any]			     =tokenizer(*_A						,      padding='''max_length'''						,      max_length=60						,      return_tensors='''np'''  )
       UpperCAmelCase			:       Union[str, Any]			     =tokenizer(_A						,      padding=_A						,      truncate=_A						,      return_tensors='''np'''  )

       # s
       # test single string max_length padding
       self.assertEqual(out_s['''input_ids'''].shape[-1]						,      30  )
       self.assertTrue(pad_token_id in out_s['''input_ids''']  )
       self.assertTrue(0 in out_s['''attention_mask''']  )

       # s2
       # test automatic padding
       self.assertEqual(out_sa['''input_ids'''].shape[-1]						,      33  )
       # long slice doesn't have padding
       self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]  )
       self.assertFalse(0 in out_sa['''attention_mask'''][0]  )
       # short slice does have padding
       self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]  )
       self.assertTrue(0 in out_sa['''attention_mask'''][1]  )

       # p
       # test single pair max_length padding
       self.assertEqual(out_p['''input_ids'''].shape[-1]						,      60  )
       self.assertTrue(pad_token_id in out_p['''input_ids''']  )
       self.assertTrue(0 in out_p['''attention_mask''']  )

       # p2
       # test automatic padding pair
       self.assertEqual(out_pa['''input_ids'''].shape[-1]						,      52  )
       # long slice pair doesn't have padding
       self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]  )
       self.assertFalse(0 in out_pa['''attention_mask'''][0]  )
       # short slice pair does have padding
       self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]  )
       self.assertTrue(0 in out_pa['''attention_mask'''][1]  )
 def 		UpperCAmelCase__				(     self  )		->							Optional[Any]:
       '''simple docstring'''





       UpperCAmelCase			:       Tuple			     ="$$$"
       UpperCAmelCase			:       str			     =CodeGenTokenizer.from_pretrained(self.tmpdirname						,      bos_token=_A						,      add_bos_token=_A  )

       UpperCAmelCase			:       str			     ="This is a simple input"
       UpperCAmelCase			:       Optional[Any]			     =["This is a simple input 1", "This is a simple input 2"]

       UpperCAmelCase			:       Tuple			     =tokenizer.bos_token_id

       UpperCAmelCase			:       List[Any]			     =tokenizer(_A  )
       UpperCAmelCase			:       Optional[int]			     =tokenizer(_A  )

       self.assertEqual(out_s.input_ids[0]						,      _A  )
       self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids  )  )

       UpperCAmelCase			:       Optional[int]			     =tokenizer.decode(out_s.input_ids  )
       UpperCAmelCase			:       int			     =tokenizer.batch_decode(out_sa.input_ids  )

       self.assertEqual(decode_s.split()[0]						,      _A  )
       self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa  )  )
 @slow
 def 		UpperCAmelCase__				(     self  )		->							Optional[int]:
       '''simple docstring'''





       UpperCAmelCase			:       Optional[Any]			     =CodeGenTokenizer.from_pretrained('''Salesforce/codegen-350M-mono'''  )

       UpperCAmelCase			:       List[str]			     ="\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"
       UpperCAmelCase			:       int			     ="\nif len_a > len_b:      result = a\nelse:      result = b"

       UpperCAmelCase			:       Optional[int]			     =tokenizer.encode(_A  )
       UpperCAmelCase			:       Union[str, Any]			     =["^#", re.escape('''<|endoftext|>'''  ), "^'''", "^\"\"\"", "\n\n\n"]
       UpperCAmelCase			:       Optional[Any]			     =tokenizer.decode(_A						,      truncate_before_pattern=_A  )
       self.assertEqual(_A						,      _A  )






 def 		UpperCAmelCase__				(     self  )		->							str:
       '''simple docstring'''





       pass

from functools import lru_cache




@lru_cache
def       _snake_case	(   lowerCAmelCase							:   int	):
		"""simple docstring"""







		if num < 0:
				raise ValueError("Number should not be negative."	)

		return 1 if num in (0, 1) else num * factorial(num - 1	)


if __name__ == "__main__":
		import doctest

		doctest.testmod()
