import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 		a__						(					A__    ,		unittest.TestCase					):
					A	       =							CodeGenTokenizer
					A	       =							CodeGenTokenizerFast
					A	       =							True
					A	       =							{'add_prefix_space': True}
					A	       =							False

					def 				__UpperCamelCase   (    self      :       Optional[int]    ):







							"""simple docstring"""






							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    [
							    "l",
							    "o",
							    "w",
							    "e",
							    "r",
							    "s",
							    "t",
							    "i",
							    "d",
							    "n",
							    "\u0120",
							    "\u0120l",
							    "\u0120n",
							    "\u0120lo",
							    "\u0120low",
							    "er",
							    "\u0120lowest",
							    "\u0120newer",
							    "\u0120wider",
							    "<unk>",
							    "<|endoftext|>",
							]
							SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    dict(zip(_A,range(len(_A    )    )    )    )
							SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
							SCREAMING_SNAKE_CASE_  :		List[str]				      =    {"unk_token": "<unk>"}

							SCREAMING_SNAKE_CASE_  :		Tuple				      =    os.path.join(self.tmpdirname,VOCAB_FILES_NAMES["vocab_file"]    )
							SCREAMING_SNAKE_CASE_  :		Any				      =    os.path.join(self.tmpdirname,VOCAB_FILES_NAMES["merges_file"]    )
							with open(self.vocab_file,"w",encoding="utf-8"    ) as fp:
									fp.write(json.dumps(_A    ) + "\n"    )
							with open(self.merges_file,"w",encoding="utf-8"    ) as fp:
									fp.write("\n".join(_A    )    )

					def 				__UpperCamelCase   (    self      :       Tuple,**_A      :       Optional[Any]    ):







							"""simple docstring"""






							kwargs.update(self.special_tokens_map    )
							return CodeGenTokenizer.from_pretrained(self.tmpdirname,**_A    )

					def 				__UpperCamelCase   (    self      :       Any,**_A      :       int    ):







							"""simple docstring"""






							kwargs.update(self.special_tokens_map    )
							return CodeGenTokenizerFast.from_pretrained(self.tmpdirname,**_A    )

					def 				__UpperCamelCase   (    self      :       Optional[Any],_A      :       Tuple    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Any				      =    "lower newer"
							SCREAMING_SNAKE_CASE_  :		Dict				      =    "lower newer"
							return input_text, output_text

					def 				__UpperCamelCase   (    self      :       List[str]    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		List[str]				      =    CodeGenTokenizer(self.vocab_file,self.merges_file,**self.special_tokens_map    )
							SCREAMING_SNAKE_CASE_  :		int				      =    "lower newer"
							SCREAMING_SNAKE_CASE_  :		List[Any]				      =    ["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
							SCREAMING_SNAKE_CASE_  :		Tuple				      =    tokenizer.tokenize(_A,add_prefix_space=_A    )
							self.assertListEqual(_A,_A    )

							SCREAMING_SNAKE_CASE_  :		int				      =    tokens + [tokenizer.unk_token]
							SCREAMING_SNAKE_CASE_  :		List[str]				      =    [14, 15, 10, 9, 3, 2, 15, 19]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_A    ),_A    )

					def 				__UpperCamelCase   (    self      :       Optional[int]    ):







							"""simple docstring"""






							if not self.test_rust_tokenizer:
									return

							SCREAMING_SNAKE_CASE_  :		Tuple				      =    self.get_tokenizer()
							SCREAMING_SNAKE_CASE_  :		Any				      =    self.get_rust_tokenizer(add_prefix_space=_A    )

							SCREAMING_SNAKE_CASE_  :		List[str]				      =    "lower newer"

							# Testing tokenization
							SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    tokenizer.tokenize(_A,add_prefix_space=_A    )
							SCREAMING_SNAKE_CASE_  :		Any				      =    rust_tokenizer.tokenize(_A    )
							self.assertListEqual(_A,_A    )

							# Testing conversion to ids without special tokens
							SCREAMING_SNAKE_CASE_  :		int				      =    tokenizer.encode(_A,add_special_tokens=_A,add_prefix_space=_A    )
							SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    rust_tokenizer.encode(_A,add_special_tokens=_A    )
							self.assertListEqual(_A,_A    )

							# Testing conversion to ids with special tokens
							SCREAMING_SNAKE_CASE_  :		int				      =    self.get_rust_tokenizer(add_prefix_space=_A    )
							SCREAMING_SNAKE_CASE_  :		int				      =    tokenizer.encode(_A,add_prefix_space=_A    )
							SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    rust_tokenizer.encode(_A    )
							self.assertListEqual(_A,_A    )

							# Testing the unknown token
							SCREAMING_SNAKE_CASE_  :		int				      =    tokens + [rust_tokenizer.unk_token]
							SCREAMING_SNAKE_CASE_  :		Any				      =    [14, 15, 10, 9, 3, 2, 15, 19]
							self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(_A    ),_A    )

					def 				__UpperCamelCase   (    self      :       Dict,*_A      :       Union[str, Any],**_A      :       Optional[int]    ):







							"""simple docstring"""






							pass

					def 				__UpperCamelCase   (    self      :       Dict,_A      :       Any=15    ):







							"""simple docstring"""






							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'    ):
											SCREAMING_SNAKE_CASE_  :		Tuple				      =    self.rust_tokenizer_class.from_pretrained(_A,**_A    )

											# Simple input
											SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    "This is a simple input"
											SCREAMING_SNAKE_CASE_  :		Dict				      =    ["This is a simple input 1", "This is a simple input 2"]
											SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    ("This is a simple input", "This is a pair")
											SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    [
											    ("This is a simple input 1", "This is a simple input 2"),
											    ("This is a simple pair 1", "This is a simple pair 2"),
											]

											# Simple input tests
											self.assertRaises(_A,tokenizer_r.encode,_A,max_length=_A,padding="max_length"    )

											# Simple input
											self.assertRaises(_A,tokenizer_r.encode_plus,_A,max_length=_A,padding="max_length"    )

											# Simple input
											self.assertRaises(
											    _A,tokenizer_r.batch_encode_plus,_A,max_length=_A,padding="max_length",)

											# Pair input
											self.assertRaises(_A,tokenizer_r.encode,_A,max_length=_A,padding="max_length"    )

											# Pair input
											self.assertRaises(_A,tokenizer_r.encode_plus,_A,max_length=_A,padding="max_length"    )

											# Pair input
											self.assertRaises(
											    _A,tokenizer_r.batch_encode_plus,_A,max_length=_A,padding="max_length",)

					def 				__UpperCamelCase   (    self      :       Tuple    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		int				      =    CodeGenTokenizer.from_pretrained(self.tmpdirname,pad_token="<pad>"    )

							# Simple input
							SCREAMING_SNAKE_CASE_  :		Any				      =    "This is a simple input"
							SCREAMING_SNAKE_CASE_  :		str				      =    ["This is a simple input looooooooong", "This is a simple input"]
							SCREAMING_SNAKE_CASE_  :		int				      =    ("This is a simple input", "This is a pair")
							SCREAMING_SNAKE_CASE_  :		List[str]				      =    [
							    ("This is a simple input loooooong", "This is a simple input"),
							    ("This is a simple pair loooooong", "This is a simple pair"),
							]

							SCREAMING_SNAKE_CASE_  :		Dict				      =    tokenizer.pad_token_id

							SCREAMING_SNAKE_CASE_  :		str				      =    tokenizer(_A,padding="max_length",max_length=30,return_tensors="np"    )
							SCREAMING_SNAKE_CASE_  :		Tuple				      =    tokenizer(_A,padding=_A,truncate=_A,return_tensors="np"    )
							SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    tokenizer(*_A,padding="max_length",max_length=60,return_tensors="np"    )
							SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    tokenizer(_A,padding=_A,truncate=_A,return_tensors="np"    )

							# s
							# test single string max_length padding
							self.assertEqual(out_s["input_ids"].shape[-1],30    )
							self.assertTrue(pad_token_id in out_s["input_ids"]    )
							self.assertTrue(0 in out_s["attention_mask"]    )

							# s2
							# test automatic padding
							self.assertEqual(out_sa["input_ids"].shape[-1],33    )
							# long slice doesn't have padding
							self.assertFalse(pad_token_id in out_sa["input_ids"][0]    )
							self.assertFalse(0 in out_sa["attention_mask"][0]    )
							# short slice does have padding
							self.assertTrue(pad_token_id in out_sa["input_ids"][1]    )
							self.assertTrue(0 in out_sa["attention_mask"][1]    )

							# p
							# test single pair max_length padding
							self.assertEqual(out_p["input_ids"].shape[-1],60    )
							self.assertTrue(pad_token_id in out_p["input_ids"]    )
							self.assertTrue(0 in out_p["attention_mask"]    )

							# p2
							# test automatic padding pair
							self.assertEqual(out_pa["input_ids"].shape[-1],52    )
							# long slice pair doesn't have padding
							self.assertFalse(pad_token_id in out_pa["input_ids"][0]    )
							self.assertFalse(0 in out_pa["attention_mask"][0]    )
							# short slice pair does have padding
							self.assertTrue(pad_token_id in out_pa["input_ids"][1]    )
							self.assertTrue(0 in out_pa["attention_mask"][1]    )

					def 				__UpperCamelCase   (    self      :       Union[str, Any]    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Tuple				      =    "$$$"
							SCREAMING_SNAKE_CASE_  :		str				      =    CodeGenTokenizer.from_pretrained(self.tmpdirname,bos_token=_A,add_bos_token=_A    )

							SCREAMING_SNAKE_CASE_  :		str				      =    "This is a simple input"
							SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    ["This is a simple input 1", "This is a simple input 2"]

							SCREAMING_SNAKE_CASE_  :		Tuple				      =    tokenizer.bos_token_id

							SCREAMING_SNAKE_CASE_  :		List[Any]				      =    tokenizer(_A    )
							SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    tokenizer(_A    )

							self.assertEqual(out_s.input_ids[0],_A    )
							self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids    )    )

							SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    tokenizer.decode(out_s.input_ids    )
							SCREAMING_SNAKE_CASE_  :		int				      =    tokenizer.batch_decode(out_sa.input_ids    )

							self.assertEqual(decode_s.split()[0],_A    )
							self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa    )    )

					@slow
					def 				__UpperCamelCase   (    self      :       Dict    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-mono"    )

							SCREAMING_SNAKE_CASE_  :		List[str]				      =    "\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"
							SCREAMING_SNAKE_CASE_  :		int				      =    "\nif len_a > len_b:      result = a\nelse:      result = b"

							SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    tokenizer.encode(_A    )
							SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    ["^#", re.escape("<|endoftext|>"    ), "^'''", "^\"\"\"", "\n\n\n"]
							SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    tokenizer.decode(_A,truncate_before_pattern=_A    )
							self.assertEqual(_A,_A    )






					def 				__UpperCamelCase   (    self      :       Tuple    ):







							"""simple docstring"""






							pass

def       _snake_case	(   lowerCAmelCase							:   int   , lowerCAmelCase							:   int	):
		"""simple docstring"""







		while b:
				SCREAMING_SNAKE_CASE_      ,    SCREAMING_SNAKE_CASE_  :		int				      =    b, a % b
		return a




def       _snake_case	(   lowerCAmelCase							:   int   , lowerCAmelCase							:   int	):
		"""simple docstring"""







		return a if b == 0 else euclidean_gcd_recursive(lowerCAmelCase   , a % b	)




def       _snake_case	(   ):
		"""simple docstring"""







		print(f'euclidean_gcd(3, 5) = {euclidean_gcd(3   , 5	)}'	)
		print(f'euclidean_gcd(5, 3) = {euclidean_gcd(5   , 3	)}'	)
		print(f'euclidean_gcd(1, 3) = {euclidean_gcd(1   , 3	)}'	)
		print(f'euclidean_gcd(3, 6) = {euclidean_gcd(3   , 6	)}'	)
		print(f'euclidean_gcd(6, 3) = {euclidean_gcd(6   , 3	)}'	)

		print(f'euclidean_gcd_recursive(3, 5) = {euclidean_gcd_recursive(3   , 5	)}'	)
		print(f'euclidean_gcd_recursive(5, 3) = {euclidean_gcd_recursive(5   , 3	)}'	)
		print(f'euclidean_gcd_recursive(1, 3) = {euclidean_gcd_recursive(1   , 3	)}'	)
		print(f'euclidean_gcd_recursive(3, 6) = {euclidean_gcd_recursive(3   , 6	)}'	)
		print(f'euclidean_gcd_recursive(6, 3) = {euclidean_gcd_recursive(6   , 3	)}'	)


if __name__ == "__main__":
		main()
