
"""simple docstring"""




import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
					if is_torch_available():
										import torch
					if is_tf_available():
										import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


UpperCAmelCase__      :    List[str]    							= logging.get_logger(__name__)

UpperCAmelCase__      :    Union[str, Any]    							= {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase__      :    Union[str, Any]    							= {
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'Salesforce/codegen-350M-mono': (
            'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'
        ),
    },
}

UpperCAmelCase__      :    int    							= {
    'Salesforce/codegen-350M-mono': 2_0_4_8,
}



class 	lowerCAmelCase_						(a__  ):


				"""simple docstring"""



				__UpperCamelCase      :						Optional[Any]					       =   VOCAB_FILES_NAMES
				__UpperCamelCase      :						Any					       =   PRETRAINED_VOCAB_FILES_MAP
				__UpperCamelCase      :						Any					       =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__UpperCamelCase      :						List[Any]					       =   ['''input_ids''', '''attention_mask''']
				__UpperCamelCase      :						Any					       =   CodeGenTokenizer



				def __init__(self   ,       SCREAMING_SNAKE_CASE__=None   ,       SCREAMING_SNAKE_CASE__=None   ,       SCREAMING_SNAKE_CASE__=None   ,       SCREAMING_SNAKE_CASE__="<|endoftext|>"   ,       SCREAMING_SNAKE_CASE__="<|endoftext|>"   ,       SCREAMING_SNAKE_CASE__="<|endoftext|>"   ,       SCREAMING_SNAKE_CASE__=False   ,       **SCREAMING_SNAKE_CASE__   ,       )					->		str:



											"""simple docstring"""




											super().__init__(
											    SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__   ,       tokenizer_file=SCREAMING_SNAKE_CASE__   ,       unk_token=SCREAMING_SNAKE_CASE__   ,       bos_token=SCREAMING_SNAKE_CASE__   ,       eos_token=SCREAMING_SNAKE_CASE__   ,       add_prefix_space=SCREAMING_SNAKE_CASE__   ,       **SCREAMING_SNAKE_CASE__   ,       )

											if kwargs.pop("""add_bos_token"""   ,       SCREAMING_SNAKE_CASE__    ):
																		SCREAMING_SNAKE_CASE__						:       Tuple           =   kwargs.pop("""name_or_path"""   ,       """"""    )
																		raise ValueError(
																		    """Currenty GPT2's fast tokenizer does NOT support adding a BOS token."""
																		    """Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"""
																		    F'''`CodeGenTokenizer.from_pretrained(\'{model_id}\')`\nor\n'''
																		    F'''`AutoTokenizer.from_pretrained(\'{model_id}\', use_fast=False)`\n'''
																		    """This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."""
																		    """ so that the fast tokenizer works correctly."""    )

											SCREAMING_SNAKE_CASE__						:       Any           =   json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()    )
											if pre_tok_state.get("""add_prefix_space"""   ,       SCREAMING_SNAKE_CASE__    ) != add_prefix_space:
																		SCREAMING_SNAKE_CASE__						:       str           =   getattr(SCREAMING_SNAKE_CASE__   ,       pre_tok_state.pop("""type"""    )    )
																		SCREAMING_SNAKE_CASE__						:       str           =   add_prefix_space
																		SCREAMING_SNAKE_CASE__						:       Any           =   pre_tok_class(**SCREAMING_SNAKE_CASE__    )

											SCREAMING_SNAKE_CASE__						:       List[Any]           =   add_prefix_space



				def 	__magic_name__    (self   ,       *SCREAMING_SNAKE_CASE__   ,       **SCREAMING_SNAKE_CASE__    )					->		BatchEncoding:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       str           =   kwargs.get("""is_split_into_words"""   ,       SCREAMING_SNAKE_CASE__    )
											assert self.add_prefix_space or not is_split_into_words, (
											    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
											    "to use it with pretokenized inputs."
											)

											return super()._batch_encode_plus(*SCREAMING_SNAKE_CASE__   ,       **SCREAMING_SNAKE_CASE__    )



				def 	__magic_name__    (self   ,       *SCREAMING_SNAKE_CASE__   ,       **SCREAMING_SNAKE_CASE__    )					->		BatchEncoding:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       str           =   kwargs.get("""is_split_into_words"""   ,       SCREAMING_SNAKE_CASE__    )

											assert self.add_prefix_space or not is_split_into_words, (
											    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
											    "to use it with pretokenized inputs."
											)

											return super()._encode_plus(*SCREAMING_SNAKE_CASE__   ,       **SCREAMING_SNAKE_CASE__    )



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__ = None    )					->		Tuple[str]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   self._tokenizer.model.save(SCREAMING_SNAKE_CASE__   ,       name=SCREAMING_SNAKE_CASE__    )
											return tuple(SCREAMING_SNAKE_CASE__    )



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__ = False   ,       SCREAMING_SNAKE_CASE__ = None   ,       SCREAMING_SNAKE_CASE__ = None   ,       **SCREAMING_SNAKE_CASE__   ,       )					->		str:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Tuple           =   super().decode(
											    token_ids=SCREAMING_SNAKE_CASE__   ,       skip_special_tokens=SCREAMING_SNAKE_CASE__   ,       clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE__   ,       **SCREAMING_SNAKE_CASE__   ,       )

											if truncate_before_pattern is not None and len(SCREAMING_SNAKE_CASE__    ) > 0:
																		SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   self.truncate(SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__    )

											return decoded_text



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__    )					->		Union[str, Any]:



											"""simple docstring"""




											def find_re(SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__    ):
																		SCREAMING_SNAKE_CASE__						:       List[Any]           =   pattern.search(SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__    )
																		return m.start() if m else -1

											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   [re.compile(SCREAMING_SNAKE_CASE__   ,       re.MULTILINE    ) for pattern in truncate_before_pattern]

											SCREAMING_SNAKE_CASE__						:       int           =   list(re.finditer("""^print"""   ,       SCREAMING_SNAKE_CASE__   ,       re.MULTILINE    )    )

											if len(SCREAMING_SNAKE_CASE__    ) > 1:
																		SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   completion[: prints[1].start()]

											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   list(re.finditer("""^def"""   ,       SCREAMING_SNAKE_CASE__   ,       re.MULTILINE    )    )

											if len(SCREAMING_SNAKE_CASE__    ) > 1:
																		SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   completion[: defs[1].start()]

											SCREAMING_SNAKE_CASE__						:       Optional[int]           =   0

											SCREAMING_SNAKE_CASE__						:       int           =   [
											    pos for pos in [find_re(SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__    ) for terminal in terminals] if pos != -1
											]

											if len(SCREAMING_SNAKE_CASE__    ) > 0:
																		return completion[: min(SCREAMING_SNAKE_CASE__    )]
											else:
																		return completion








from typing import TYPE_CHECKING

from ..utils import _LazyModule


UpperCAmelCase__   	=							{
    '''config''': [
        '''EXTERNAL_DATA_FORMAT_SIZE_LIMIT''',
        '''OnnxConfig''',
        '''OnnxConfigWithPast''',
        '''OnnxSeq2SeqConfigWithPast''',
        '''PatchingSpec''',
    ],
    '''convert''': ['''export''', '''validate_model_outputs'''],
    '''features''': ['''FeaturesManager'''],
    '''utils''': ['''ParameterFormat''', '''compute_serialized_parameters_size'''],
}


if TYPE_CHECKING:
  from .config import (
      EXTERNAL_DATA_FORMAT_SIZE_LIMIT,
      OnnxConfig,
      OnnxConfigWithPast,
      OnnxSeqaSeqConfigWithPast,
      PatchingSpec,
  )
  from .convert import export, validate_model_outputs
  from .features import FeaturesManager
  from .utils import ParameterFormat, compute_serialized_parameters_size

else:
  import sys

  UpperCAmelCase__   	=							_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)

