

'''simple docstring'''



import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
  if is_torch_available():
    import torch
  if is_tf_available():
    import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


UpperCAmelCase :       List[Any]			   =     logging.get_logger(__name__)

UpperCAmelCase :       str			   =     {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase :       Any			   =     {
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'Salesforce/codegen-350M-mono': (
            'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'
        ),
    },
}

UpperCAmelCase :       Tuple			   =     {
    'Salesforce/codegen-350M-mono': 2_0_4_8,
}







class 						lowerCAmelCase__   (      A__					):



 """simple docstring"""



 lowerCAmelCase__  					=      VOCAB_FILES_NAMES
 lowerCAmelCase__  					=      PRETRAINED_VOCAB_FILES_MAP
 lowerCAmelCase__  					=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowerCAmelCase__  					=      ['''input_ids''', '''attention_mask''']
 lowerCAmelCase__  					=      CodeGenTokenizer
 def __init__(		self					:       List[str]					,							__SCREAMING_SNAKE_CASE					:       Dict=None					,							__SCREAMING_SNAKE_CASE					:       List[str]=None					,							__SCREAMING_SNAKE_CASE					:       Any=None					,							__SCREAMING_SNAKE_CASE					:       Optional[int]="<|endoftext|>"					,							__SCREAMING_SNAKE_CASE					:       Optional[int]="<|endoftext|>"					,							__SCREAMING_SNAKE_CASE					:       str="<|endoftext|>"					,							__SCREAMING_SNAKE_CASE					:       Any=False					,							**__SCREAMING_SNAKE_CASE					:       Dict					,							)  -> Union[str, Any]:


      """simple docstring"""




      super().__init__(
          lowerCAmelCase__					,							lowerCAmelCase__					,							tokenizer_file=lowerCAmelCase__					,							unk_token=lowerCAmelCase__					,							bos_token=lowerCAmelCase__					,							eos_token=lowerCAmelCase__					,							add_prefix_space=lowerCAmelCase__					,							**lowerCAmelCase__					,							)

      if kwargs.pop("""add_bos_token"""					,							lowerCAmelCase__			):
           __SCREAMING_SNAKE_CASE     =      kwargs.pop("""name_or_path"""					,							""""""			)
           raise ValueError(
               """Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token."""
               """Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n"""
               f'`CodeGenTokenizer.from_pretrained(\'{model_id}\')`\nor\n'
               f'`AutoTokenizer.from_pretrained(\'{model_id}\', use_fast=False)`\n'
               """This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."""
               """ so that the fast tokenizer works correctly."""			)

      __SCREAMING_SNAKE_CASE     =      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()			)
      if pre_tok_state.get("""add_prefix_space"""					,							lowerCAmelCase__			) != add_prefix_space:
           __SCREAMING_SNAKE_CASE     =      getattr(lowerCAmelCase__					,							pre_tok_state.pop("""type"""			)			)
           __SCREAMING_SNAKE_CASE     =      add_prefix_space
           __SCREAMING_SNAKE_CASE     =      pre_tok_class(**lowerCAmelCase__			)

      __SCREAMING_SNAKE_CASE     =      add_prefix_space
 def       UpperCAmelCase__		(		self					:       Optional[Any]					,							*__SCREAMING_SNAKE_CASE					:       Optional[int]					,							**__SCREAMING_SNAKE_CASE					:       List[Any]			)  -> BatchEncoding:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      kwargs.get("""is_split_into_words"""					,							lowerCAmelCase__			)
      assert self.add_prefix_space or not is_split_into_words, (
          f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
          "to use it with pretokenized inputs."
      )

      return super()._batch_encode_plus(*lowerCAmelCase__					,							**lowerCAmelCase__			)
 def       UpperCAmelCase__		(		self					:       List[Any]					,							*__SCREAMING_SNAKE_CASE					:       Optional[Any]					,							**__SCREAMING_SNAKE_CASE					:       List[Any]			)  -> BatchEncoding:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      kwargs.get("""is_split_into_words"""					,							lowerCAmelCase__			)

      assert self.add_prefix_space or not is_split_into_words, (
          f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
          "to use it with pretokenized inputs."
      )

      return super()._encode_plus(*lowerCAmelCase__					,							**lowerCAmelCase__			)
 def       UpperCAmelCase__		(		self					:       List[str]					,							__SCREAMING_SNAKE_CASE					:       str					,							__SCREAMING_SNAKE_CASE					:       Optional[str] = None			)  -> Tuple[str]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self._tokenizer.model.save(lowerCAmelCase__					,							name=lowerCAmelCase__			)
      return tuple(lowerCAmelCase__			)
 def       UpperCAmelCase__		(		self					:       Tuple					,							__SCREAMING_SNAKE_CASE					:       Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]					,							__SCREAMING_SNAKE_CASE					:       bool = False					,							__SCREAMING_SNAKE_CASE					:       bool = None					,							__SCREAMING_SNAKE_CASE					:       Optional[List[str]] = None					,							**__SCREAMING_SNAKE_CASE					:       Optional[int]					,							)  -> str:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      super().decode(
          token_ids=lowerCAmelCase__					,							skip_special_tokens=lowerCAmelCase__					,							clean_up_tokenization_spaces=lowerCAmelCase__					,							**lowerCAmelCase__					,							)

      if truncate_before_pattern is not None and len(lowerCAmelCase__			) > 0:
           __SCREAMING_SNAKE_CASE     =      self.truncate(lowerCAmelCase__					,							lowerCAmelCase__			)

      return decoded_text


 def       UpperCAmelCase__		(		self					:       int					,							__SCREAMING_SNAKE_CASE					:       Any					,							__SCREAMING_SNAKE_CASE					:       Union[str, Any]			)  -> Dict:


      """simple docstring"""




      def find_re(__SCREAMING_SNAKE_CASE					:       int					,							__SCREAMING_SNAKE_CASE					:       Optional[int]					,							__SCREAMING_SNAKE_CASE					:       int			):
           __SCREAMING_SNAKE_CASE     =      pattern.search(lowerCAmelCase__					,							lowerCAmelCase__			)
           return m.start() if m else -1

      __SCREAMING_SNAKE_CASE     =      [re.compile(lowerCAmelCase__					,							re.MULTILINE			) for pattern in truncate_before_pattern]

      __SCREAMING_SNAKE_CASE     =      list(re.finditer("""^print"""					,							lowerCAmelCase__					,							re.MULTILINE			)			)

      if len(lowerCAmelCase__			) > 1:
           __SCREAMING_SNAKE_CASE     =      completion[: prints[1].start()]

      __SCREAMING_SNAKE_CASE     =      list(re.finditer("""^def"""					,							lowerCAmelCase__					,							re.MULTILINE			)			)

      if len(lowerCAmelCase__			) > 1:
           __SCREAMING_SNAKE_CASE     =      completion[: defs[1].start()]

      __SCREAMING_SNAKE_CASE     =      0

      __SCREAMING_SNAKE_CASE     =      [
          pos for pos in [find_re(lowerCAmelCase__					,							lowerCAmelCase__					,							lowerCAmelCase__			) for terminal in terminals] if pos != -1
      ]

      if len(lowerCAmelCase__			) > 0:
           return completion[: min(lowerCAmelCase__			)]
      else:
           return completion










from typing import Optional

from .. import Features, NamedSplit
from ..packaged_modules.text.text import Text
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader
class        _lowercase      (			A__       ):






				'''simple docstring'''



				def __init__(       self			:int			,			lowerCAmelCase__			:NestedDataStructureLike[PathLike]			,			lowerCAmelCase__			:Optional[NamedSplit] = None			,			lowerCAmelCase__			:Optional[Features] = None			,			lowerCAmelCase__			:str = None			,			lowerCAmelCase__			:bool = False			,			lowerCAmelCase__			:bool = False			,			lowerCAmelCase__			:Optional[int] = None			,			**lowerCAmelCase__			:Optional[int]			,			)		->			Tuple:
										super().__init__(
										    lowerCAmelCase__			,			split=lowerCAmelCase__			,			features=lowerCAmelCase__			,			cache_dir=lowerCAmelCase__			,			keep_in_memory=lowerCAmelCase__			,			streaming=lowerCAmelCase__			,			num_proc=lowerCAmelCase__			,			**lowerCAmelCase__			,			)
										__SCREAMING_SNAKE_CASE						:     List[str]					=		path_or_paths if isinstance(lowerCAmelCase__			,			lowerCAmelCase__	) else {self.split: path_or_paths}
										__SCREAMING_SNAKE_CASE						:     int					=		Text(
										    cache_dir=lowerCAmelCase__			,			data_files=lowerCAmelCase__			,			features=lowerCAmelCase__			,			**lowerCAmelCase__			,			)



				def 		__magic_name__(       self			:Dict	)		->			Tuple:
										# Build iterable dataset
										if self.streaming:
																__SCREAMING_SNAKE_CASE						:     int					=		self.builder.as_streaming_dataset(split=self.split	)
										# Build regular (map-style) dataset
										else:
																__SCREAMING_SNAKE_CASE						:     List[str]					=		None
																__SCREAMING_SNAKE_CASE						:     str					=		None
																__SCREAMING_SNAKE_CASE						:     Dict					=		None
																__SCREAMING_SNAKE_CASE						:     Tuple					=		None

																self.builder.download_and_prepare(
																    download_config=lowerCAmelCase__			,			download_mode=lowerCAmelCase__			,			verification_mode=lowerCAmelCase__			,			base_path=lowerCAmelCase__			,			num_proc=self.num_proc			,			)
																__SCREAMING_SNAKE_CASE						:     Optional[int]					=		self.builder.as_dataset(
																    split=self.split			,			verification_mode=lowerCAmelCase__			,			in_memory=self.keep_in_memory	)
										return dataset
