





"""simple docstring"""
import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 					_UpperCAmelCase   (	a__     ,							unittest.TestCase ):
					UpperCamelCase      =							CodeGenTokenizer
					UpperCamelCase      =							CodeGenTokenizerFast
					UpperCamelCase      =							True
					UpperCamelCase      =							{'''add_prefix_space''': True}
					UpperCamelCase      =							False


					def 	lowerCamelCase       (					self			:Tuple		):
												super().setUp()

												# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
												A		=  [
												    """l""",
												    """o""",
												    """w""",
												    """e""",
												    """r""",
												    """s""",
												    """t""",
												    """i""",
												    """d""",
												    """n""",
												    """\u0120""",
												    """\u0120l""",
												    """\u0120n""",
												    """\u0120lo""",
												    """\u0120low""",
												    """er""",
												    """\u0120lowest""",
												    """\u0120newer""",
												    """\u0120wider""",
												    """<unk>""",
												    """<|endoftext|>""",
												]
												A		=  dict(zip(SCREAMING_SNAKE_CASE__     ,       range(len(SCREAMING_SNAKE_CASE__		)		)		)		)
												A		=  ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
												A		=  {"""unk_token""": """<unk>"""}

												A		=  os.path.join(self.tmpdirname     ,       VOCAB_FILES_NAMES["vocab_file"]		)
												A		=  os.path.join(self.tmpdirname     ,       VOCAB_FILES_NAMES["merges_file"]		)
												with open(self.vocab_file     ,       "w"     ,       encoding="utf-8"		) as fp:
																			fp.write(json.dumps(SCREAMING_SNAKE_CASE__		) + "\n"		)
												with open(self.merges_file     ,       "w"     ,       encoding="utf-8"		) as fp:
																			fp.write("\n".join(SCREAMING_SNAKE_CASE__		)		)


					def 	lowerCamelCase       (					self			:Dict     ,       **__UpperCamelCase			:Tuple		):
												kwargs.update(self.special_tokens_map		)
												return CodeGenTokenizer.from_pretrained(self.tmpdirname     ,       **SCREAMING_SNAKE_CASE__		)


					def 	lowerCamelCase       (					self			:List[str]     ,       **__UpperCamelCase			:Union[str, Any]		):
												kwargs.update(self.special_tokens_map		)
												return CodeGenTokenizerFast.from_pretrained(self.tmpdirname     ,       **SCREAMING_SNAKE_CASE__		)


					def 	lowerCamelCase       (					self			:Dict     ,       __UpperCamelCase			:Union[str, Any]		):
												A		=  """lower newer"""
												A		=  """lower newer"""
												return input_text, output_text


					def 	lowerCamelCase       (					self			:Dict		):
												A		=  CodeGenTokenizer(self.vocab_file     ,       self.merges_file     ,       **self.special_tokens_map		)
												A		=  """lower newer"""
												A		=  ["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
												A		=  tokenizer.tokenize(SCREAMING_SNAKE_CASE__     ,       add_prefix_space=SCREAMING_SNAKE_CASE__		)
												self.assertListEqual(SCREAMING_SNAKE_CASE__     ,       SCREAMING_SNAKE_CASE__		)

												A		=  tokens + [tokenizer.unk_token]
												A		=  [14, 15, 10, 9, 3, 2, 15, 19]
												self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__		)     ,       SCREAMING_SNAKE_CASE__		)


					def 	lowerCamelCase       (					self			:int		):




												if not self.test_rust_tokenizer:
																			return

												A		=  self.get_tokenizer()
												A		=  self.get_rust_tokenizer(add_prefix_space=SCREAMING_SNAKE_CASE__		)

												A		=  """lower newer"""

												# Testing tokenization
												A		=  tokenizer.tokenize(SCREAMING_SNAKE_CASE__     ,       add_prefix_space=SCREAMING_SNAKE_CASE__		)
												A		=  rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE__		)
												self.assertListEqual(SCREAMING_SNAKE_CASE__     ,       SCREAMING_SNAKE_CASE__		)

												# Testing conversion to ids without special tokens
												A		=  tokenizer.encode(SCREAMING_SNAKE_CASE__     ,       add_special_tokens=SCREAMING_SNAKE_CASE__     ,       add_prefix_space=SCREAMING_SNAKE_CASE__		)
												A		=  rust_tokenizer.encode(SCREAMING_SNAKE_CASE__     ,       add_special_tokens=SCREAMING_SNAKE_CASE__		)
												self.assertListEqual(SCREAMING_SNAKE_CASE__     ,       SCREAMING_SNAKE_CASE__		)

												# Testing conversion to ids with special tokens
												A		=  self.get_rust_tokenizer(add_prefix_space=SCREAMING_SNAKE_CASE__		)
												A		=  tokenizer.encode(SCREAMING_SNAKE_CASE__     ,       add_prefix_space=SCREAMING_SNAKE_CASE__		)
												A		=  rust_tokenizer.encode(SCREAMING_SNAKE_CASE__		)
												self.assertListEqual(SCREAMING_SNAKE_CASE__     ,       SCREAMING_SNAKE_CASE__		)

												# Testing the unknown token
												A		=  tokens + [rust_tokenizer.unk_token]
												A		=  [14, 15, 10, 9, 3, 2, 15, 19]
												self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__		)     ,       SCREAMING_SNAKE_CASE__		)


					def 	lowerCamelCase       (					self			:List[str]     ,       *__UpperCamelCase			:List[Any]     ,       **__UpperCamelCase			:Union[str, Any]		):
												pass


					def 	lowerCamelCase       (					self			:int     ,       __UpperCamelCase			:str=15		):




												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"		):
																										A		=  self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__     ,       **SCREAMING_SNAKE_CASE__		)

																										# Simple input
																										A		=  """This is a simple input"""
																										A		=  ["""This is a simple input 1""", """This is a simple input 2"""]
																										A		=  ("""This is a simple input""", """This is a pair""")
																										A		=  [
																										    ("""This is a simple input 1""", """This is a simple input 2"""),
																										    ("""This is a simple pair 1""", """This is a simple pair 2"""),
																										]

																										# Simple input tests
																										self.assertRaises(SCREAMING_SNAKE_CASE__     ,       tokenizer_r.encode     ,       SCREAMING_SNAKE_CASE__     ,       max_length=SCREAMING_SNAKE_CASE__     ,       padding="max_length"		)

																										# Simple input
																										self.assertRaises(SCREAMING_SNAKE_CASE__     ,       tokenizer_r.encode_plus     ,       SCREAMING_SNAKE_CASE__     ,       max_length=SCREAMING_SNAKE_CASE__     ,       padding="max_length"		)

																										# Simple input
																										self.assertRaises(
																										    SCREAMING_SNAKE_CASE__     ,       tokenizer_r.batch_encode_plus     ,       SCREAMING_SNAKE_CASE__     ,       max_length=SCREAMING_SNAKE_CASE__     ,       padding="max_length"     ,       )

																										# Pair input
																										self.assertRaises(SCREAMING_SNAKE_CASE__     ,       tokenizer_r.encode     ,       SCREAMING_SNAKE_CASE__     ,       max_length=SCREAMING_SNAKE_CASE__     ,       padding="max_length"		)

																										# Pair input
																										self.assertRaises(SCREAMING_SNAKE_CASE__     ,       tokenizer_r.encode_plus     ,       SCREAMING_SNAKE_CASE__     ,       max_length=SCREAMING_SNAKE_CASE__     ,       padding="max_length"		)

																										# Pair input
																										self.assertRaises(
																										    SCREAMING_SNAKE_CASE__     ,       tokenizer_r.batch_encode_plus     ,       SCREAMING_SNAKE_CASE__     ,       max_length=SCREAMING_SNAKE_CASE__     ,       padding="max_length"     ,       )


					def 	lowerCamelCase       (					self			:Optional[int]		):
												A		=  CodeGenTokenizer.from_pretrained(self.tmpdirname     ,       pad_token="<pad>"		)

												# Simple input
												A		=  """This is a simple input"""
												A		=  ["""This is a simple input looooooooong""", """This is a simple input"""]
												A		=  ("""This is a simple input""", """This is a pair""")
												A		=  [
												    ("""This is a simple input loooooong""", """This is a simple input"""),
												    ("""This is a simple pair loooooong""", """This is a simple pair"""),
												]

												A		=  tokenizer.pad_token_id

												A		=  tokenizer(SCREAMING_SNAKE_CASE__     ,       padding="max_length"     ,       max_length=30     ,       return_tensors="np"		)
												A		=  tokenizer(SCREAMING_SNAKE_CASE__     ,       padding=SCREAMING_SNAKE_CASE__     ,       truncate=SCREAMING_SNAKE_CASE__     ,       return_tensors="np"		)
												A		=  tokenizer(*SCREAMING_SNAKE_CASE__     ,       padding="max_length"     ,       max_length=60     ,       return_tensors="np"		)
												A		=  tokenizer(SCREAMING_SNAKE_CASE__     ,       padding=SCREAMING_SNAKE_CASE__     ,       truncate=SCREAMING_SNAKE_CASE__     ,       return_tensors="np"		)

												# s
												# test single string max_length padding
												self.assertEqual(out_s["input_ids"].shape[-1]     ,       30		)
												self.assertTrue(pad_token_id in out_s["input_ids"]		)
												self.assertTrue(0 in out_s["attention_mask"]		)

												# s2
												# test automatic padding
												self.assertEqual(out_sa["input_ids"].shape[-1]     ,       33		)
												# long slice doesn't have padding
												self.assertFalse(pad_token_id in out_sa["input_ids"][0]		)
												self.assertFalse(0 in out_sa["attention_mask"][0]		)
												# short slice does have padding
												self.assertTrue(pad_token_id in out_sa["input_ids"][1]		)
												self.assertTrue(0 in out_sa["attention_mask"][1]		)

												# p
												# test single pair max_length padding
												self.assertEqual(out_p["input_ids"].shape[-1]     ,       60		)
												self.assertTrue(pad_token_id in out_p["input_ids"]		)
												self.assertTrue(0 in out_p["attention_mask"]		)

												# p2
												# test automatic padding pair
												self.assertEqual(out_pa["input_ids"].shape[-1]     ,       52		)
												# long slice pair doesn't have padding
												self.assertFalse(pad_token_id in out_pa["input_ids"][0]		)
												self.assertFalse(0 in out_pa["attention_mask"][0]		)
												# short slice pair does have padding
												self.assertTrue(pad_token_id in out_pa["input_ids"][1]		)
												self.assertTrue(0 in out_pa["attention_mask"][1]		)


					def 	lowerCamelCase       (					self			:List[Any]		):
												A		=  """$$$"""
												A		=  CodeGenTokenizer.from_pretrained(self.tmpdirname     ,       bos_token=SCREAMING_SNAKE_CASE__     ,       add_bos_token=SCREAMING_SNAKE_CASE__		)

												A		=  """This is a simple input"""
												A		=  ["""This is a simple input 1""", """This is a simple input 2"""]

												A		=  tokenizer.bos_token_id

												A		=  tokenizer(SCREAMING_SNAKE_CASE__		)
												A		=  tokenizer(SCREAMING_SNAKE_CASE__		)

												self.assertEqual(out_s.input_ids[0]     ,       SCREAMING_SNAKE_CASE__		)
												self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids		)		)

												A		=  tokenizer.decode(out_s.input_ids		)
												A		=  tokenizer.batch_decode(out_sa.input_ids		)

												self.assertEqual(decode_s.split()[0]     ,       SCREAMING_SNAKE_CASE__		)
												self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa		)		)


					@slow
					def 	lowerCamelCase       (					self			:List[str]		):
												A		=  CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-mono"		)

												A		=  """\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
												A		=  """\nif len_a > len_b:      result = a\nelse:      result = b"""

												A		=  tokenizer.encode(SCREAMING_SNAKE_CASE__		)
												A		=  ["""^#""", re.escape("<|endoftext|>"		), """^'''""", """^\"\"\"""", """\n\n\n"""]
												A		=  tokenizer.decode(SCREAMING_SNAKE_CASE__     ,       truncate_before_pattern=SCREAMING_SNAKE_CASE__		)
												self.assertEqual(SCREAMING_SNAKE_CASE__     ,       SCREAMING_SNAKE_CASE__		)


					def 	lowerCamelCase       (					self			:Tuple		):
												pass




"""simple docstring"""




import logging
import os
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

from tqdm import auto as tqdm_lib


UpperCAmelCase__      :    Optional[int]    							= {
    'debug': logging.DEBUG,
    'info': logging.INFO,
    'warning': logging.WARNING,
    'error': logging.ERROR,
    'critical': logging.CRITICAL,
}

UpperCAmelCase__      :    List[Any]    							= logging.WARNING

def lowercase_  (   ):
							SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   os.getenv("""DATASETS_VERBOSITY"""   ,_snake_case						)
							if env_level_str:
														if env_level_str in log_levels:
																					return log_levels[env_level_str]
														else:
																					logging.getLogger().warning(
																					    f'''Unknown option DATASETS_VERBOSITY={env_level_str}, '''
																					    f'''has to be one of: { ', '.join(log_levels.keys()						) }'''						)
							return _default_log_level

def lowercase_  (   ):
							return __name__.split("""."""						)[0]

def lowercase_  (   ):
							return logging.getLogger(_get_library_name()						)

def lowercase_  (   ):
							# Apply our default configuration to the library root logger.
							SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   _get_library_root_logger()
							library_root_logger.setLevel(_get_default_logging_level()						)

def lowercase_  (   ):
							SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   _get_library_root_logger()
							library_root_logger.setLevel(logging.NOTSET						)

def lowercase_  (   _snake_case = None						):
							if name is None:
														SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   _get_library_name()
							return logging.getLogger(_snake_case						)

def lowercase_  (   ):
							return _get_library_root_logger().getEffectiveLevel()

def lowercase_  (   _snake_case						):
							_get_library_root_logger().setLevel(_snake_case						)

def lowercase_  (   ):
							return set_verbosity(_snake_case						)

def lowercase_  (   ):
							return set_verbosity(_snake_case						)

def lowercase_  (   ):
							return set_verbosity(_snake_case						)

def lowercase_  (   ):
							return set_verbosity(_snake_case						)

def lowercase_  (   ):
							SCREAMING_SNAKE_CASE__						:       Tuple           =   False

def lowercase_  (   ):
							SCREAMING_SNAKE_CASE__						:       str           =   True


# Configure the library root logger at the module level (singleton-like)
_configure_library_root_logger()



class 	lowerCAmelCase_						:


				"""simple docstring"""



				def __init__(self   ,       *SCREAMING_SNAKE_CASE__   ,       **SCREAMING_SNAKE_CASE__    )					->		int:  # pylint: disable=unused-argument



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       List[str]           =   args[0] if args else None



				def __iter__(self    )					->		int:



											"""simple docstring"""




											return iter(self._iterator    )



				def __getattr__(self   ,       SCREAMING_SNAKE_CASE__    )					->		int:



											"""simple docstring"""




											def empty_fn(*SCREAMING_SNAKE_CASE__   ,       **SCREAMING_SNAKE_CASE__    ):  # pylint: disable=unused-argument
																		return

											return empty_fn



				def __enter__(self    )					->		Dict:



											"""simple docstring"""




											return self



				def __exit__(self   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__    )					->		str:



											"""simple docstring"""




											return






UpperCAmelCase__      :    str    							= True



class 	lowerCAmelCase_						:


				"""simple docstring"""



				def __call__(self   ,       *SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__=False   ,       **SCREAMING_SNAKE_CASE__    )					->		List[Any]:



											"""simple docstring"""




											if _tqdm_active and not disable:
																		return tqdm_lib.tqdm(*SCREAMING_SNAKE_CASE__   ,       **SCREAMING_SNAKE_CASE__    )
											else:
																		return EmptyTqdm(*SCREAMING_SNAKE_CASE__   ,       **SCREAMING_SNAKE_CASE__    )



				def 	__magic_name__    (self   ,       *SCREAMING_SNAKE_CASE__   ,       **SCREAMING_SNAKE_CASE__    )					->		int:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       List[Any]           =   None
											if _tqdm_active:
																		return tqdm_lib.tqdm.set_lock(*SCREAMING_SNAKE_CASE__   ,       **SCREAMING_SNAKE_CASE__    )



				def 	__magic_name__    (self    )					->		Optional[Any]:



											"""simple docstring"""




											if _tqdm_active:
																		return tqdm_lib.tqdm.get_lock()






UpperCAmelCase__      :    Tuple    							= _tqdm_cls()

def lowercase_  (   ):
							global _tqdm_active
							return bool(_tqdm_active						)

def lowercase_  (   ):
							global _tqdm_active
							SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   True

def lowercase_  (   ):
							global _tqdm_active
							SCREAMING_SNAKE_CASE__						:       str           =   False




