


import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
						if is_torch_available():
												import torch
						if is_tf_available():
												import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


__lowerCAmelCase	: List[Any]  				=logging.get_logger(__name__)

__lowerCAmelCase	: str  				={'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

__lowerCAmelCase	: Any  				={
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'Salesforce/codegen-350M-mono': (
            'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'
        ),
    },
}

__lowerCAmelCase	: Tuple  				={
    'Salesforce/codegen-350M-mono': 2_0_4_8,
}
class        _lowercase      (			A__       ):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE__				:			Dict						=		VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE__				:			Optional[Any]						=		PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE__				:			Optional[Any]						=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE__				:			str						=		['''input_ids''', '''attention_mask''']
				SCREAMING_SNAKE_CASE__				:			str						=		CodeGenTokenizer


				def __init__(       self			:List[str]			,			lowerCAmelCase__			:Dict=None			,			lowerCAmelCase__			:List[str]=None			,			lowerCAmelCase__			:Any=None			,			lowerCAmelCase__			:Optional[int]="<|endoftext|>"			,			lowerCAmelCase__			:Optional[int]="<|endoftext|>"			,			lowerCAmelCase__			:str="<|endoftext|>"			,			lowerCAmelCase__			:Any=False			,			**lowerCAmelCase__			:Dict			,			)		->			Union[str, Any]:
										super().__init__(
										    lowerCAmelCase__			,			lowerCAmelCase__			,			tokenizer_file=lowerCAmelCase__			,			unk_token=lowerCAmelCase__			,			bos_token=lowerCAmelCase__			,			eos_token=lowerCAmelCase__			,			add_prefix_space=lowerCAmelCase__			,			**lowerCAmelCase__			,			)

										if kwargs.pop('''add_bos_token'''			,			lowerCAmelCase__	):
																__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		kwargs.pop('''name_or_path'''			,			''''''	)
																raise ValueError(
																    '''Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'''
																    '''Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'''
																    f'''`CodeGenTokenizer.from_pretrained(\'{model_id}\')`\nor\n'''
																    f'''`AutoTokenizer.from_pretrained(\'{model_id}\', use_fast=False)`\n'''
																    '''This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'''
																    ''' so that the fast tokenizer works correctly.'''	)

										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()	)
										if pre_tok_state.get('''add_prefix_space'''			,			lowerCAmelCase__	) != add_prefix_space:
																__SCREAMING_SNAKE_CASE						:     int					=		getattr(lowerCAmelCase__			,			pre_tok_state.pop('''type'''	)	)
																__SCREAMING_SNAKE_CASE						:     Any					=		add_prefix_space
																__SCREAMING_SNAKE_CASE						:     Dict					=		pre_tok_class(**lowerCAmelCase__	)

										__SCREAMING_SNAKE_CASE						:     Tuple					=		add_prefix_space


				def 		__magic_name__(       self			:Optional[Any]			,			*lowerCAmelCase__			:Optional[int]			,			**lowerCAmelCase__			:List[Any]	)		->			BatchEncoding:
										__SCREAMING_SNAKE_CASE						:     str					=		kwargs.get('''is_split_into_words'''			,			lowerCAmelCase__	)
										assert self.add_prefix_space or not is_split_into_words, (
										    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
										    "to use it with pretokenized inputs."
										)

										return super()._batch_encode_plus(*lowerCAmelCase__			,			**lowerCAmelCase__	)


				def 		__magic_name__(       self			:List[Any]			,			*lowerCAmelCase__			:Optional[Any]			,			**lowerCAmelCase__			:List[Any]	)		->			BatchEncoding:
										__SCREAMING_SNAKE_CASE						:     List[Any]					=		kwargs.get('''is_split_into_words'''			,			lowerCAmelCase__	)

										assert self.add_prefix_space or not is_split_into_words, (
										    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
										    "to use it with pretokenized inputs."
										)

										return super()._encode_plus(*lowerCAmelCase__			,			**lowerCAmelCase__	)


				def 		__magic_name__(       self			:List[str]			,			lowerCAmelCase__			:str			,			lowerCAmelCase__			:Optional[str] = None	)		->			Tuple[str]:
										__SCREAMING_SNAKE_CASE						:     Dict					=		self._tokenizer.model.save(lowerCAmelCase__			,			name=lowerCAmelCase__	)
										return tuple(lowerCAmelCase__	)


				def 		__magic_name__(       self			:Tuple			,			lowerCAmelCase__			:Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]			,			lowerCAmelCase__			:bool = False			,			lowerCAmelCase__			:bool = None			,			lowerCAmelCase__			:Optional[List[str]] = None			,			**lowerCAmelCase__			:Optional[int]			,			)		->			str:
										__SCREAMING_SNAKE_CASE						:     str					=		super().decode(
										    token_ids=lowerCAmelCase__			,			skip_special_tokens=lowerCAmelCase__			,			clean_up_tokenization_spaces=lowerCAmelCase__			,			**lowerCAmelCase__			,			)

										if truncate_before_pattern is not None and len(lowerCAmelCase__	) > 0:
																__SCREAMING_SNAKE_CASE						:     List[str]					=		self.truncate(lowerCAmelCase__			,			lowerCAmelCase__	)

										return decoded_text



				def 		__magic_name__(       self			:int			,			lowerCAmelCase__			:Any			,			lowerCAmelCase__			:Union[str, Any]	)		->			Dict:
										def find_re(lowerCAmelCase__			:int			,			lowerCAmelCase__			:Optional[int]			,			lowerCAmelCase__			:int	):
																__SCREAMING_SNAKE_CASE						:     List[Any]					=		pattern.search(lowerCAmelCase__			,			lowerCAmelCase__	)
																return m.start() if m else -1

										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		[re.compile(lowerCAmelCase__			,			re.MULTILINE	) for pattern in truncate_before_pattern]

										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		list(re.finditer('''^print'''			,			lowerCAmelCase__			,			re.MULTILINE	)	)

										if len(lowerCAmelCase__	) > 1:
																__SCREAMING_SNAKE_CASE						:     Optional[int]					=		completion[: prints[1].start()]

										__SCREAMING_SNAKE_CASE						:     int					=		list(re.finditer('''^def'''			,			lowerCAmelCase__			,			re.MULTILINE	)	)

										if len(lowerCAmelCase__	) > 1:
																__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		completion[: defs[1].start()]

										__SCREAMING_SNAKE_CASE						:     str					=		0

										__SCREAMING_SNAKE_CASE						:     str					=		[
										    pos for pos in [find_re(lowerCAmelCase__			,			lowerCAmelCase__			,			lowerCAmelCase__	) for terminal in terminals] if pos != -1
										]

										if len(lowerCAmelCase__	) > 0:
																return completion[: min(lowerCAmelCase__	)]
										else:
																return completion




import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin
class        _lowercase      (			A__						,			unittest.TestCase       ):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE__				:			Optional[int]						=		TransfoXLTokenizer
				SCREAMING_SNAKE_CASE__				:			int						=		False
				SCREAMING_SNAKE_CASE__				:			Optional[Any]						=		False


				def 		__magic_name__(       self			:str	)		->			Dict:
										super().setUp()

										__SCREAMING_SNAKE_CASE						:     List[str]					=		[
										    '''<unk>''',
										    '''[CLS]''',
										    '''[SEP]''',
										    '''want''',
										    '''unwanted''',
										    '''wa''',
										    '''un''',
										    '''running''',
										    ''',''',
										    '''low''',
										    '''l''',
										]
										__SCREAMING_SNAKE_CASE						:     Any					=		os.path.join(self.tmpdirname			,			VOCAB_FILES_NAMES['''vocab_file''']	)
										with open(self.vocab_file			,			'''w'''			,			encoding='''utf-8'''	) as vocab_writer:
																vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]	)	)


				def 		__magic_name__(       self			:Any			,			**lowerCAmelCase__			:int	)		->			str:
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		True
										return TransfoXLTokenizer.from_pretrained(self.tmpdirname			,			**lowerCAmelCase__	)


				def 		__magic_name__(       self			:Union[str, Any]			,			lowerCAmelCase__			:List[Any]	)		->			Tuple:
										__SCREAMING_SNAKE_CASE						:     Dict					=		'''<unk> UNwanted , running'''
										__SCREAMING_SNAKE_CASE						:     List[str]					=		'''<unk> unwanted, running'''
										return input_text, output_text


				def 		__magic_name__(       self			:Any	)		->			Union[str, Any]:
										__SCREAMING_SNAKE_CASE						:     int					=		TransfoXLTokenizer(vocab_file=self.vocab_file			,			lower_case=lowerCAmelCase__	)

										__SCREAMING_SNAKE_CASE						:     int					=		tokenizer.tokenize('''<unk> UNwanted , running'''	)
										self.assertListEqual(lowerCAmelCase__			,			['''<unk>''', '''unwanted''', ''',''', '''running''']	)

										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__	)			,			[0, 4, 8, 7]	)


				def 		__magic_name__(       self			:Tuple	)		->			Union[str, Any]:
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		TransfoXLTokenizer(lower_case=lowerCAmelCase__	)

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo ! how  \n Are yoU ?  '''	)			,			['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']	)


				def 		__magic_name__(       self			:Tuple	)		->			List[Any]:
										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		TransfoXLTokenizer(lower_case=lowerCAmelCase__	)

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo ! how  \n Are yoU ?  '''	)			,			['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)


				def 		__magic_name__(       self			:Dict	)		->			List[Any]:
										__SCREAMING_SNAKE_CASE						:     List[str]					=		TransfoXLTokenizer(lower_case=lowerCAmelCase__	)
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		'''Hello (bracket) and side-scrolled [and] Henry\'s $5,000 with 3.34 m. What\'s up!?'''
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		[
										    '''Hello''',
										    '''(''',
										    '''bracket''',
										    ''')''',
										    '''and''',
										    '''side''',
										    '''@-@''',
										    '''scrolled''',
										    '''[''',
										    '''and''',
										    ''']''',
										    '''Henry''',
										    '''\'s''',
										    '''$''',
										    '''5''',
										    '''@,@''',
										    '''000''',
										    '''with''',
										    '''3''',
										    '''@.@''',
										    '''34''',
										    '''m''',
										    '''.''',
										    '''What''',
										    '''\'s''',
										    '''up''',
										    '''!''',
										    '''?''',
										]

										self.assertListEqual(tokenizer.tokenize(lowerCAmelCase__	)			,			lowerCAmelCase__	)

										self.assertEqual(tokenizer.convert_tokens_to_string(lowerCAmelCase__	)			,			lowerCAmelCase__	)



				def 		__magic_name__(       self			:str	)		->			int:
										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		self.get_tokenizer()
										__SCREAMING_SNAKE_CASE						:     Any					=		len(lowerCAmelCase__	)

										tokenizer.add_tokens(['''new1''', '''new2''']	)
										tokenizer.move_added_token('''new1'''			,			1	)

										# Check that moved token is not copied (duplicate)
										self.assertEqual(len(lowerCAmelCase__	)			,			original_len + 2	)
										# Check that token is moved to specified id
										self.assertEqual(tokenizer.encode('''new1'''	)			,			[1]	)
										self.assertEqual(tokenizer.decode([1]	)			,			'''new1'''	)
