


"""simple docstring"""







import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
 if is_torch_available():
  import torch
 if is_tf_available():
  import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


__lowercase          = logging.get_logger(__name__)

__lowercase          = {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

__lowercase          = {
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
}

__lowercase          = {
    'Salesforce/codegen-350M-mono': 2048,
}


@lru_cache()
def 	lowercase     (							)-> int:
  '''simple docstring'''







  a		:						Dict 				=     (
      list(range(ord("!"			)    ,  ord("~"			) + 1			)			) + list(range(ord("¡"			)    ,  ord("¬"			) + 1			)			) + list(range(ord("®"			)    ,  ord("ÿ"			) + 1			)			)
  )
  a		:						Union[str, Any] 				=     bs[:]
  a		:						str 				=     0
  for b in range(2**8			):
    if b not in bs:
      bs.append(__UpperCamelCase			)
      cs.append(2**8 + n			)
      n += 1
  a		:						Dict 				=     [chr(__UpperCamelCase			) for n in cs]
  return dict(zip(__UpperCamelCase    ,  __UpperCamelCase			)			)


def 	lowercase     (							A_			)-> Union[str, Any]:
  '''simple docstring'''







  a		:						Dict 				=     set()
  a		:						str 				=     word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)			)
    a		:						int 				=     char
  return pairs







class 		_A    (    SCREAMING_SNAKE_CASE_    ):







    """simple docstring"""




    UpperCAmelCase			:				List[str]      =			VOCAB_FILES_NAMES
    UpperCAmelCase			:				Tuple      =			PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase			:				List[Any]      =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCAmelCase			:				List[Any]      =			["""input_ids""", """attention_mask"""]






    def __init__(   self							:       Dict							,					__UpperCAmelCase							:       List[Any]							,					__UpperCAmelCase							:       List[Any]							,					__UpperCAmelCase							:       int="replace"							,					__UpperCAmelCase							:       Union[str, Any]="<|endoftext|>"							,					__UpperCAmelCase							:       Union[str, Any]="<|endoftext|>"							,					__UpperCAmelCase							:       int="<|endoftext|>"							,					__UpperCAmelCase							:       Tuple=None							,					__UpperCAmelCase							:       Union[str, Any]=False							,					__UpperCAmelCase							:       Union[str, Any]=False							,					**__UpperCAmelCase							:       Any							,					):
      a		:						str 				=     AddedToken(UpperCamelCase__							,					lstrip=UpperCamelCase__							,					rstrip=UpperCamelCase__) if isinstance(UpperCamelCase__							,					UpperCamelCase__) else bos_token
      a		:						List[Any] 				=     AddedToken(UpperCamelCase__							,					lstrip=UpperCamelCase__							,					rstrip=UpperCamelCase__) if isinstance(UpperCamelCase__							,					UpperCamelCase__) else eos_token
      a		:						Any 				=     AddedToken(UpperCamelCase__							,					lstrip=UpperCamelCase__							,					rstrip=UpperCamelCase__) if isinstance(UpperCamelCase__							,					UpperCamelCase__) else unk_token
      a		:						Dict 				=     AddedToken(UpperCamelCase__							,					lstrip=UpperCamelCase__							,					rstrip=UpperCamelCase__) if isinstance(UpperCamelCase__							,					UpperCamelCase__) else pad_token
      super().__init__(
          errors=UpperCamelCase__							,					unk_token=UpperCamelCase__							,					bos_token=UpperCamelCase__							,					eos_token=UpperCamelCase__							,					pad_token=UpperCamelCase__							,					add_prefix_space=UpperCamelCase__							,					add_bos_token=UpperCamelCase__							,					**UpperCamelCase__							,					)
      a		:						int 				=     add_bos_token

      with open(UpperCamelCase__							,					encoding="utf-8") as vocab_handle:
        a		:						Tuple 				=     json.load(UpperCamelCase__)
      a		:						Union[str, Any] 				=     {v: k for k, v in self.encoder.items()}
      a		:						List[Any] 				=     errors  # how to handle errors in decoding
      a		:						List[Any] 				=     bytes_to_unicode()
      a		:						Union[str, Any] 				=     {v: k for k, v in self.byte_encoder.items()}
      with open(UpperCamelCase__							,					encoding="utf-8") as merges_handle:
        a		:						List[str] 				=     merges_handle.read().split("\n")[1:-1]
      a		:						Any 				=     [tuple(merge.split()) for merge in bpe_merges]
      a		:						Any 				=     dict(zip(UpperCamelCase__							,					range(len(UpperCamelCase__))))
      a		:						int 				=     {}
      a		:						Dict 				=     add_prefix_space

      # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
      a		:						List[str] 				=     re.compile(r"\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+")






    @property
    def       __snake_case				(   self							:       str):
      return len(self.encoder)






    def       __snake_case				(   self							:       Union[str, Any]):
      return dict(self.encoder							,					**self.added_tokens_encoder)






    def       __snake_case				(   self							:       Any							,					__UpperCAmelCase							:       Union[str, Any]):
      if token in self.cache:
        return self.cache[token]
      a		:						Union[str, Any] 				=     tuple(UpperCamelCase__)
      a		:						List[str] 				=     get_pairs(UpperCamelCase__)

      if not pairs:
        return token

      while True:
        a		:						Optional[int] 				=     min(UpperCamelCase__							,					key=lambda __UpperCAmelCase: self.bpe_ranks.get(UpperCamelCase__							,					float("inf")))
        if bigram not in self.bpe_ranks:
          break
        a		:						str 				=     bigram
        a		:						Union[str, Any] 				=     []
        a		:						List[str] 				=     0
        while i < len(UpperCamelCase__):
          try:
            a		:						List[str] 				=     word.index(UpperCamelCase__							,					UpperCamelCase__)
          except ValueError:
            new_word.extend(word[i:])
            break
          else:
            new_word.extend(word[i:j])
            a		:						Optional[int] 				=     j

          if word[i] == first and i < len(UpperCamelCase__) - 1 and word[i + 1] == second:
            new_word.append(first + second)
            i += 2
          else:
            new_word.append(word[i])
            i += 1
        a		:						Union[str, Any] 				=     tuple(UpperCamelCase__)
        a		:						Optional[Any] 				=     new_word
        if len(UpperCamelCase__) == 1:
          break
        else:
          a		:						int 				=     get_pairs(UpperCamelCase__)
      a		:						Tuple 				=     ''' '''.join(UpperCamelCase__)
      a		:						Optional[int] 				=     word
      return word






    def       __snake_case				(   self							:       Optional[int]							,					__UpperCAmelCase							:       Tuple							,					__UpperCAmelCase							:       Any=None):
      if self.add_bos_token:
        a		:						Any 				=     [self.bos_token_id]
      else:
        a		:						List[str] 				=     []

      a		:						str 				=     bos_token_ids + token_ids_a

      if token_ids_a is None:
        return output

      return output + bos_token_ids + token_ids_a






    def       __snake_case				(   self							:       Tuple							,					__UpperCAmelCase							:       Optional[int]):
      a		:						List[Any] 				=     []
      for token in re.findall(self.pat							,					UpperCamelCase__):
        a		:						List[str] 				=     ''''''.join(
            self.byte_encoder[b] for b in token.encode("utf-8"))  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
        bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCamelCase__).split(" "))
      return bpe_tokens






    def       __snake_case				(   self							:       Optional[Any]							,					__UpperCAmelCase							:       Tuple):
      return self.encoder.get(UpperCamelCase__							,					self.encoder.get(self.unk_token))






    def       __snake_case				(   self							:       Tuple							,					__UpperCAmelCase							:       str):
      return self.decoder.get(UpperCamelCase__)






    def       __snake_case				(   self							:       List[str]							,					__UpperCAmelCase							:       Dict):
      a		:						Dict 				=     ''''''.join(UpperCamelCase__)
      a		:						Union[str, Any] 				=     bytearray([self.byte_decoder[c] for c in text]).decode("utf-8"							,					errors=self.errors)
      return text






    def       __snake_case				(   self							:       Union[str, Any]							,					__UpperCAmelCase							:       Union[str, Any]							,					__UpperCAmelCase							:       List[str] = None):
      if not os.path.isdir(UpperCamelCase__):
        logger.error(f'''Vocabulary path ({save_directory}) should be a directory''')
        return
      a		:						Union[str, Any] 				=     os.path.join(
          UpperCamelCase__							,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])
      a		:						Optional[Any] 				=     os.path.join(
          UpperCamelCase__							,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"])

      with open(UpperCamelCase__							,					"w"							,					encoding="utf-8") as f:
        f.write(json.dumps(self.encoder							,					indent=2							,					sort_keys=UpperCamelCase__							,					ensure_ascii=UpperCamelCase__) + "\n")

      a		:						Union[str, Any] 				=     0
      with open(UpperCamelCase__							,					"w"							,					encoding="utf-8") as writer:
        writer.write("#version: 0.2\n")
        for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,					key=lambda __UpperCAmelCase: kv[1]):
          if index != token_index:
            logger.warning(
                f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                " Please check that the tokenizer is not corrupted!")
            a		:						Union[str, Any] 				=     token_index
          writer.write(" ".join(UpperCamelCase__) + "\n")
          index += 1

      return vocab_file, merge_file






    def       __snake_case				(   self							:       Optional[int]							,					__UpperCAmelCase							:       List[Any]							,					__UpperCAmelCase							:       Dict=False							,					**__UpperCAmelCase							:       Any):
      a		:						str 				=     kwargs.pop("add_prefix_space"							,					self.add_prefix_space)
      if is_split_into_words or add_prefix_space:
        a		:						Tuple 				=     ''' ''' + text
      return (text, kwargs)






    def       __snake_case				(   self							:       int							,					__UpperCAmelCase							:       str							,					__UpperCAmelCase							:       Optional[Any] = False							,					__UpperCAmelCase							:       Union[str, Any] = None							,					__UpperCAmelCase							:       Optional[Any] = None							,					**__UpperCAmelCase							:       List[Any]							,					):
      a		:						int 				=     super()._decode(
          token_ids=UpperCamelCase__							,					skip_special_tokens=UpperCamelCase__							,					clean_up_tokenization_spaces=UpperCamelCase__							,					**UpperCamelCase__							,					)

      if truncate_before_pattern is not None and len(UpperCamelCase__) > 0:
        a		:						Union[str, Any] 				=     self.truncate(UpperCamelCase__							,					UpperCamelCase__)

      return decoded_text






    def       __snake_case				(   self							:       Optional[int]							,					__UpperCAmelCase							:       List[str]							,					__UpperCAmelCase							:       Any):
      def find_re(__UpperCAmelCase							:       Optional[Any]							,					__UpperCAmelCase							:       str							,					__UpperCAmelCase							:       Optional[Any]):
        a		:						int 				=     pattern.search(UpperCamelCase__							,					UpperCamelCase__)
        return m.start() if m else -1

      a		:						Optional[Any] 				=     [re.compile(UpperCamelCase__							,					re.MULTILINE) for pattern in truncate_before_pattern]

      a		:						int 				=     list(re.finditer("^print"							,					UpperCamelCase__							,					re.MULTILINE))

      if len(UpperCamelCase__) > 1:
        a		:						List[str] 				=     completion[: prints[1].start()]

      a		:						Union[str, Any] 				=     list(re.finditer("^def"							,					UpperCamelCase__							,					re.MULTILINE))

      if len(UpperCamelCase__) > 1:
        a		:						Any 				=     completion[: defs[1].start()]

      a		:						str 				=     0

      a		:						List[str] 				=     [
          pos for pos in [find_re(UpperCamelCase__							,					UpperCamelCase__							,					UpperCamelCase__) for terminal in terminals] if pos != -1
      ]

      if len(UpperCamelCase__) > 0:
        return completion[: min(UpperCamelCase__)]
      else:
        return completion








import os
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device


if is_torch_available():
					import torch

					from transformers import GPTaLMHeadModel







@require_torch
@require_sentencepiece
@require_tokenizers
class 			UpperCamelCase__		(     unittest.TestCase ):







	'''simple docstring'''





	@slow
	@unittest.skip('''Model is not available.'''		)
	def 	__snake_case					(      self		):
					A__	:						int								=				'''nvidia/megatron-gpt2-345m/'''
					if "MYDIR" in os.environ:
									A__	:						Union[str, Any]								=				os.path.join(os.environ['''MYDIR''']					,							UpperCamelCase__		)
					A__	:						Union[str, Any]								=				GPTaLMHeadModel.from_pretrained(UpperCamelCase__		)
					model.to(UpperCamelCase__		)
					model.half()

					A__	:						Dict								=				torch.tensor(
					    [[101, 7110, 1005, 1056, 2023, 1_1333, 1_7413, 1029, 102]]					,							device=UpperCamelCase__					,							dtype=torch.long					,							)

					with torch.no_grad():
									A__	:						str								=				model(UpperCamelCase__		).logits

					A__	:						int								=				torch.Size((1, 9, 5_0257)		)
					self.assertEqual(output.shape					,							UpperCamelCase__		)

					A__	:						Dict								=				torch.tensor(
					    [
					        4.9_4_1_4,
					        -0.2_9_2_0,
					        -1.2_1_4_8,
					        -4.0_2_7_3,
					        -0.5_1_6_1,
					        -5.2_1_0_9,
					        -1.2_4_1_2,
					        -1.8_3_0_1,
					        -1.7_7_3_4,
					        -4.7_1_4_8,
					        -0.2_3_1_7,
					        -1.0_8_1_1,
					        -2.1_7_7_7,
					        0.4_1_4_1,
					        -3.7_9_6_9,
					        -4.0_5_8_6,
					        -2.5_3_3_2,
					        -3.3_8_0_9,
					        4.3_8_6_7,
					    ]					,							device=UpperCamelCase__					,							dtype=torch.half					,							)

					for i in range(19		):
									A__							,				A__	:						int								=				8 * i // 17, 2792 * i  # along the diagonal
									A__							,				A__	:						Optional[Any]								=				output[0, r, c], expected_diag[i]
									A__	:						Dict								=				F"row={r} col={c} computed={computed} expected={expected}"
									self.assertAlmostEqual(UpperCamelCase__					,							UpperCamelCase__					,							delta=1e-4					,							msg=UpperCamelCase__		)