


'''simple docstring'''

import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
	if is_torch_available():
		import torch
	if is_tf_available():
		import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


_UpperCAmelCase :       List[Any]										= logging.get_logger(__name__)

_UpperCAmelCase :       Optional[int]										= {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_UpperCAmelCase :       str										= {
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''Salesforce/codegen-350M-mono''': (
            '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'''
        ),
    },
}

_UpperCAmelCase :       Union[str, Any]										= {
    '''Salesforce/codegen-350M-mono''': 2_0_4_8,
}


class 					a__ (       a__					):







				"""simple docstring"""



				__UpperCamelCase			:						Any		=		VOCAB_FILES_NAMES
				__UpperCamelCase			:						Optional[int]		=		PRETRAINED_VOCAB_FILES_MAP
				__UpperCamelCase			:						str		=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__UpperCamelCase			:						List[Any]		=		['input_ids', 'attention_mask']
				__UpperCamelCase			:						Union[str, Any]		=		CodeGenTokenizer




				def __init__(self				,	__lowercase=None				,	__lowercase=None				,	__lowercase=None				,	__lowercase="<|endoftext|>"				,	__lowercase="<|endoftext|>"				,	__lowercase="<|endoftext|>"				,	__lowercase=False				,	**__lowercase				,	):
										super().__init__(
										    UpperCamelCase_				,	UpperCamelCase_				,	tokenizer_file=UpperCamelCase_				,	unk_token=UpperCamelCase_				,	bos_token=UpperCamelCase_				,	eos_token=UpperCamelCase_				,	add_prefix_space=UpperCamelCase_				,	**UpperCamelCase_				,	)

										if kwargs.pop('''add_bos_token'''				,	UpperCamelCase_						):
																__lowerCAmelCase											= kwargs.pop('''name_or_path'''				,	''''''						)
																raise ValueError(
																    '''Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'''
																    '''Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'''
																    F"""`CodeGenTokenizer.from_pretrained(\'{model_id}\')`\nor\n"""
																    F"""`AutoTokenizer.from_pretrained(\'{model_id}\', use_fast=False)`\n"""
																    '''This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'''
																    ''' so that the fast tokenizer works correctly.'''						)

										__lowerCAmelCase											= json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()						)
										if pre_tok_state.get('''add_prefix_space'''				,	UpperCamelCase_						) != add_prefix_space:
																__lowerCAmelCase											= getattr(UpperCamelCase_				,	pre_tok_state.pop('''type'''						)						)
																__lowerCAmelCase											= add_prefix_space
																__lowerCAmelCase											= pre_tok_class(**UpperCamelCase_						)

										__lowerCAmelCase											= add_prefix_space




				def     _snake_case     (self				,	*__lowercase				,	**__lowercase						):
										__lowerCAmelCase											= kwargs.get('''is_split_into_words'''				,	UpperCamelCase_						)
										assert self.add_prefix_space or not is_split_into_words, (
										    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
										    "to use it with pretokenized inputs."
										)

										return super()._batch_encode_plus(*UpperCamelCase_				,	**UpperCamelCase_						)




				def     _snake_case     (self				,	*__lowercase				,	**__lowercase						):
										__lowerCAmelCase											= kwargs.get('''is_split_into_words'''				,	UpperCamelCase_						)

										assert self.add_prefix_space or not is_split_into_words, (
										    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
										    "to use it with pretokenized inputs."
										)

										return super()._encode_plus(*UpperCamelCase_				,	**UpperCamelCase_						)




				def     _snake_case     (self				,	__lowercase				,	__lowercase = None						):
										__lowerCAmelCase											= self._tokenizer.model.save(UpperCamelCase_				,	name=UpperCamelCase_						)
										return tuple(UpperCamelCase_						)




				def     _snake_case     (self				,	__lowercase				,	__lowercase = False				,	__lowercase = None				,	__lowercase = None				,	**__lowercase				,	):
										__lowerCAmelCase											= super().decode(
										    token_ids=UpperCamelCase_				,	skip_special_tokens=UpperCamelCase_				,	clean_up_tokenization_spaces=UpperCamelCase_				,	**UpperCamelCase_				,	)

										if truncate_before_pattern is not None and len(UpperCamelCase_						) > 0:
																__lowerCAmelCase											= self.truncate(UpperCamelCase_				,	UpperCamelCase_						)

										return decoded_text




				def     _snake_case     (self				,	__lowercase				,	__lowercase						):
										def find_re(__lowercase				,	__lowercase				,	__lowercase						):
																__lowerCAmelCase											= pattern.search(UpperCamelCase_				,	UpperCamelCase_						)
																return m.start() if m else -1

										__lowerCAmelCase											= [re.compile(UpperCamelCase_				,	re.MULTILINE						) for pattern in truncate_before_pattern]

										__lowerCAmelCase											= list(re.finditer('''^print'''				,	UpperCamelCase_				,	re.MULTILINE						)						)

										if len(UpperCamelCase_						) > 1:
																__lowerCAmelCase											= completion[: prints[1].start()]

										__lowerCAmelCase											= list(re.finditer('''^def'''				,	UpperCamelCase_				,	re.MULTILINE						)						)

										if len(UpperCamelCase_						) > 1:
																__lowerCAmelCase											= completion[: defs[1].start()]

										__lowerCAmelCase											= 0

										__lowerCAmelCase											= [
										    pos for pos in [find_re(UpperCamelCase_				,	UpperCamelCase_				,	UpperCamelCase_						) for terminal in terminals] if pos != -1
										]

										if len(UpperCamelCase_						) > 0:
																return completion[: min(UpperCamelCase_						)]
										else:
																return completion





"""simple docstring"""





import inspect
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
snake_case__						:   Optional[Any]			 = '''src/transformers'''


# This is to make sure the transformers module imported is the one in the repo.
snake_case__						:   Dict			 = direct_transformers_import(PATH_TO_TRANSFORMERS)

snake_case__						:   Optional[int]			 = transformers.models.auto.configuration_auto.CONFIG_MAPPING

# Regex pattern used to find the checkpoint mentioned in the docstring of `config_class`.
# For example, `[bert-base-uncased](https://huggingface.co/bert-base-uncased)`
snake_case__						:   Optional[int]			 = re.compile(R'''\[(.+?)\]\((https://huggingface\.co/.+?)\)''')


snake_case__						:   int			 = {
    '''DecisionTransformerConfig''',
    '''EncoderDecoderConfig''',
    '''MusicgenConfig''',
    '''RagConfig''',
    '''SpeechEncoderDecoderConfig''',
    '''TimmBackboneConfig''',
    '''VisionEncoderDecoderConfig''',
    '''VisionTextDualEncoderConfig''',
    '''LlamaConfig''',
}


def    _snake_case  (  _snake_case :	List[str]	):
      lowerCAmelCase    :		Dict				=      None

      # source code of `config_class`
      lowerCAmelCase    :		Union[str, Any]				=      inspect.getsource(_snake_case	)
      lowerCAmelCase    :		List[Any]				=      _re_checkpoint.findall(_snake_case	)

      # Each `checkpoint` is a tuple of a checkpoint name and a checkpoint link.
      # For example, `('bert-base-uncased', 'https://huggingface.co/bert-base-uncased')`
      for ckpt_name, ckpt_link in checkpoints:
            # allow the link to end with `/`
            if ckpt_link.endswith('''/'''	):
                  lowerCAmelCase    :		List[str]				=      ckpt_link[:-1]

            # verify the checkpoint name corresponds to the checkpoint link
            lowerCAmelCase    :		Optional[int]				=      f'''https://huggingface.co/{ckpt_name}'''
            if ckpt_link == ckpt_link_from_name:
                  lowerCAmelCase    :		List[str]				=      ckpt_name
                  break

      return checkpoint


def    _snake_case  (  ):
      lowerCAmelCase    :		List[Any]				=      []

      for config_class in list(CONFIG_MAPPING.values()	):
            # Skip deprecated models
            if "models.deprecated" in config_class.__module__:
                  continue
            lowerCAmelCase    :		int				=      get_checkpoint_from_config_class(_snake_case	)

            lowerCAmelCase    :		int				=      config_class.__name__
            if checkpoint is None and name not in CONFIG_CLASSES_TO_IGNORE_FOR_DOCSTRING_CHECKPOINT_CHECK:
                  configs_without_checkpoint.append(_snake_case	)

      if len(_snake_case	) > 0:
            lowerCAmelCase    :		Dict				=      '''\n'''.join(sorted(_snake_case	)	)
            raise ValueError(f'''The following configurations don\'t contain any valid checkpoint:\n{message}'''	)


if __name__ == "__main__":
 check_config_docstrings_have_checkpoints()


