




import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
    if is_torch_available():
        import torch
    if is_tf_available():
        import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


snake_case_					:					Union[str, Any]       						=			logging.get_logger(__name__)

snake_case_					:					Optional[Any]       						=			{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

snake_case_					:					Optional[int]       						=			{
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
}

snake_case_					:					Optional[Any]       						=			{
    "Salesforce/codegen-350M-mono": 2048,
}


@lru_cache()
def   A		()  -> Tuple:







     """simple docstring"""


     UpperCAmelCase_		=				(
         list(range(ord('''!'''			)  ,							ord('''~'''			) + 1			)			) + list(range(ord('''¡'''			)  ,							ord('''¬'''			) + 1			)			) + list(range(ord('''®'''			)  ,							ord('''ÿ'''			) + 1			)			)
     )
     UpperCAmelCase_		=				bs[:]
     UpperCAmelCase_		=				0
     for b in range(2**8			):
          if b not in bs:
               bs.append(__A			)
               cs.append(2**8 + n			)
               n += 1
     UpperCAmelCase_		=				[chr(__A			) for n in cs]
     return dict(zip(__A  ,							__A			)			)


def   A		(__A  :       Optional[Any]			)  -> Optional[int]:







     """simple docstring"""


     UpperCAmelCase_		=				set()
     UpperCAmelCase_		=				word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)			)
          UpperCAmelCase_		=				char
     return pairs







class __snake_case     (     a						):
   UpperCAmelCase__    :       Tuple		=    VOCAB_FILES_NAMES
   UpperCAmelCase__    :       str		=    PRETRAINED_VOCAB_FILES_MAP
   UpperCAmelCase__    :       Dict		=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   UpperCAmelCase__    :       Optional[int]		=    ['''input_ids''', '''attention_mask''']







   def __init__( self						:		Optional[Any]   ,		_snake_case						:		Dict   ,		_snake_case						:		Optional[int]   ,		_snake_case						:		int="replace"   ,		_snake_case						:		str="<|endoftext|>"   ,		_snake_case						:		int="<|endoftext|>"   ,		_snake_case						:		Optional[int]="<|endoftext|>"   ,		_snake_case						:		Optional[int]=None   ,		_snake_case						:		int=False   ,		_snake_case						:		Tuple=False   ,		**_snake_case						:		Any   ,		):

        """simple docstring"""


        UpperCAmelCase_		=				AddedToken(_snake_case   ,		lstrip=_snake_case   ,		rstrip=_snake_case) if isinstance(_snake_case   ,		_snake_case) else bos_token
        UpperCAmelCase_		=				AddedToken(_snake_case   ,		lstrip=_snake_case   ,		rstrip=_snake_case) if isinstance(_snake_case   ,		_snake_case) else eos_token
        UpperCAmelCase_		=				AddedToken(_snake_case   ,		lstrip=_snake_case   ,		rstrip=_snake_case) if isinstance(_snake_case   ,		_snake_case) else unk_token
        UpperCAmelCase_		=				AddedToken(_snake_case   ,		lstrip=_snake_case   ,		rstrip=_snake_case) if isinstance(_snake_case   ,		_snake_case) else pad_token
        super().__init__(
            errors=_snake_case   ,		unk_token=_snake_case   ,		bos_token=_snake_case   ,		eos_token=_snake_case   ,		pad_token=_snake_case   ,		add_prefix_space=_snake_case   ,		add_bos_token=_snake_case   ,		**_snake_case   ,		)
        UpperCAmelCase_		=				add_bos_token

        with open(_snake_case   ,		encoding='''utf-8''') as vocab_handle:
             UpperCAmelCase_		=				json.load(_snake_case)
        UpperCAmelCase_		=				{v: k for k, v in self.encoder.items()}
        UpperCAmelCase_		=				errors  # how to handle errors in decoding
        UpperCAmelCase_		=				bytes_to_unicode()
        UpperCAmelCase_		=				{v: k for k, v in self.byte_encoder.items()}
        with open(_snake_case   ,		encoding='''utf-8''') as merges_handle:
             UpperCAmelCase_		=				merges_handle.read().split('''\n''')[1:-1]
        UpperCAmelCase_		=				[tuple(merge.split()) for merge in bpe_merges]
        UpperCAmelCase_		=				dict(zip(_snake_case   ,		range(len(_snake_case))))
        UpperCAmelCase_		=				{}
        UpperCAmelCase_		=				add_prefix_space

        # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
        UpperCAmelCase_		=				re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+''')







   @property
   def lowerCamelCase		( self						:		Dict):

        """simple docstring"""


        return len(self.encoder)







   def lowerCamelCase		( self						:		str):

        """simple docstring"""


        return dict(self.encoder   ,		**self.added_tokens_encoder)







   def lowerCamelCase		( self						:		Any   ,		_snake_case						:		Optional[Any]):

        """simple docstring"""


        if token in self.cache:
             return self.cache[token]
        UpperCAmelCase_		=				tuple(_snake_case)
        UpperCAmelCase_		=				get_pairs(_snake_case)

        if not pairs:
             return token

        while True:
             UpperCAmelCase_		=				min(_snake_case   ,		key=lambda _snake_case: self.bpe_ranks.get(_snake_case   ,		float('''inf''')))
             if bigram not in self.bpe_ranks:
                  break
             UpperCAmelCase_	,			UpperCAmelCase_		=				bigram
             UpperCAmelCase_		=				[]
             UpperCAmelCase_		=				0
             while i < len(_snake_case):
                  try:
                       UpperCAmelCase_		=				word.index(_snake_case   ,		_snake_case)
                  except ValueError:
                       new_word.extend(word[i:])
                       break
                  else:
                       new_word.extend(word[i:j])
                       UpperCAmelCase_		=				j

                  if word[i] == first and i < len(_snake_case) - 1 and word[i + 1] == second:
                       new_word.append(first + second)
                       i += 2
                  else:
                       new_word.append(word[i])
                       i += 1
             UpperCAmelCase_		=				tuple(_snake_case)
             UpperCAmelCase_		=				new_word
             if len(_snake_case) == 1:
                  break
             else:
                  UpperCAmelCase_		=				get_pairs(_snake_case)
        UpperCAmelCase_		=				''' '''.join(_snake_case)
        UpperCAmelCase_		=				word
        return word







   def lowerCamelCase		( self						:		Optional[int]   ,		_snake_case						:		Optional[int]   ,		_snake_case						:		Optional[Any]=None):

        """simple docstring"""


        if self.add_bos_token:
             UpperCAmelCase_		=				[self.bos_token_id]
        else:
             UpperCAmelCase_		=				[]

        UpperCAmelCase_		=				bos_token_ids + token_ids_a

        if token_ids_a is None:
             return output

        return output + bos_token_ids + token_ids_a







   def lowerCamelCase		( self						:		List[Any]   ,		_snake_case						:		List[str]):

        """simple docstring"""


        UpperCAmelCase_		=				[]
        for token in re.findall(self.pat   ,		_snake_case):
             UpperCAmelCase_		=				''''''.join(
                 self.byte_encoder[b] for b in token.encode('''utf-8'''))  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
             bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_snake_case).split(''' '''))
        return bpe_tokens







   def lowerCamelCase		( self						:		Dict   ,		_snake_case						:		List[Any]):

        """simple docstring"""


        return self.encoder.get(_snake_case   ,		self.encoder.get(self.unk_token))







   def lowerCamelCase		( self						:		str   ,		_snake_case						:		Optional[Any]):

        """simple docstring"""


        return self.decoder.get(_snake_case)







   def lowerCamelCase		( self						:		str   ,		_snake_case						:		Dict):

        """simple docstring"""


        UpperCAmelCase_		=				''''''.join(_snake_case)
        UpperCAmelCase_		=				bytearray([self.byte_decoder[c] for c in text]).decode('''utf-8'''   ,		errors=self.errors)
        return text







   def lowerCamelCase		( self						:		int   ,		_snake_case						:		str   ,		_snake_case						:		Optional[str] = None):

        """simple docstring"""


        if not os.path.isdir(_snake_case):
             logger.error(F"""Vocabulary path ({save_directory}) should be a directory""")
             return
        UpperCAmelCase_		=				os.path.join(
            _snake_case   ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])
        UpperCAmelCase_		=				os.path.join(
            _snake_case   ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file'''])

        with open(_snake_case   ,		'''w'''   ,		encoding='''utf-8''') as f:
             f.write(json.dumps(self.encoder   ,		indent=2   ,		sort_keys=_snake_case   ,		ensure_ascii=_snake_case) + '''\n''')

        UpperCAmelCase_		=				0
        with open(_snake_case   ,		'''w'''   ,		encoding='''utf-8''') as writer:
             writer.write('''#version: 0.2\n''')
             for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,		key=lambda _snake_case: kv[1]):
                  if index != token_index:
                       logger.warning(
                           F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                           ''' Please check that the tokenizer is not corrupted!''')
                       UpperCAmelCase_		=				token_index
                  writer.write(''' '''.join(_snake_case) + '''\n''')
                  index += 1

        return vocab_file, merge_file







   def lowerCamelCase		( self						:		Optional[int]   ,		_snake_case						:		List[Any]   ,		_snake_case						:		Tuple=False   ,		**_snake_case						:		List[str]):

        """simple docstring"""


        UpperCAmelCase_		=				kwargs.pop('''add_prefix_space'''   ,		self.add_prefix_space)
        if is_split_into_words or add_prefix_space:
             UpperCAmelCase_		=				''' ''' + text
        return (text, kwargs)







   def lowerCamelCase		( self						:		List[Any]   ,		_snake_case						:		Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]   ,		_snake_case						:		bool = False   ,		_snake_case						:		bool = None   ,		_snake_case						:		Optional[List[str]] = None   ,		**_snake_case						:		List[Any]   ,		):

        """simple docstring"""


        UpperCAmelCase_		=				super()._decode(
            token_ids=_snake_case   ,		skip_special_tokens=_snake_case   ,		clean_up_tokenization_spaces=_snake_case   ,		**_snake_case   ,		)

        if truncate_before_pattern is not None and len(_snake_case) > 0:
             UpperCAmelCase_		=				self.truncate(_snake_case   ,		_snake_case)

        return decoded_text







   def lowerCamelCase		( self						:		List[str]   ,		_snake_case						:		List[str]   ,		_snake_case						:		str):

        """simple docstring"""


        def find_re(_snake_case						:		Union[str, Any]   ,		_snake_case						:		Any   ,		_snake_case						:		Optional[int]):
             UpperCAmelCase_		=				pattern.search(_snake_case   ,		_snake_case)
             return m.start() if m else -1

        UpperCAmelCase_		=				[re.compile(_snake_case   ,		re.MULTILINE) for pattern in truncate_before_pattern]

        UpperCAmelCase_		=				list(re.finditer('''^print'''   ,		_snake_case   ,		re.MULTILINE))

        if len(_snake_case) > 1:
             UpperCAmelCase_		=				completion[: prints[1].start()]

        UpperCAmelCase_		=				list(re.finditer('''^def'''   ,		_snake_case   ,		re.MULTILINE))

        if len(_snake_case) > 1:
             UpperCAmelCase_		=				completion[: defs[1].start()]

        UpperCAmelCase_		=				0

        UpperCAmelCase_		=				[
            pos for pos in [find_re(_snake_case   ,		_snake_case   ,		_snake_case) for terminal in terminals] if pos != -1
        ]

        if len(_snake_case) > 0:
             return completion[: min(_snake_case)]
        else:
             return completion












def   A		(__A  :       float  ,							__A  :       int = 2			)  -> float:







     """simple docstring"""


     return round((float(__A			) * 9 / 5) + 32  ,							__A			)


def   A		(__A  :       float  ,							__A  :       int = 2			)  -> float:







     """simple docstring"""


     return round(float(__A			) + 273.15  ,							__A			)


def   A		(__A  :       float  ,							__A  :       int = 2			)  -> float:







     """simple docstring"""


     return round((float(__A			) * 9 / 5) + 491.67  ,							__A			)


def   A		(__A  :       float  ,							__A  :       int = 2			)  -> float:







     """simple docstring"""


     return round((float(__A			) - 32) * 5 / 9  ,							__A			)


def   A		(__A  :       float  ,							__A  :       int = 2			)  -> float:







     """simple docstring"""


     return round(((float(__A			) - 32) * 5 / 9) + 273.15  ,							__A			)


def   A		(__A  :       float  ,							__A  :       int = 2			)  -> float:







     """simple docstring"""


     return round(float(__A			) + 459.67  ,							__A			)


def   A		(__A  :       float  ,							__A  :       int = 2			)  -> float:







     """simple docstring"""


     return round(float(__A			) - 273.15  ,							__A			)


def   A		(__A  :       float  ,							__A  :       int = 2			)  -> float:







     """simple docstring"""


     return round(((float(__A			) - 273.15) * 9 / 5) + 32  ,							__A			)


def   A		(__A  :       float  ,							__A  :       int = 2			)  -> float:







     """simple docstring"""


     return round((float(__A			) * 9 / 5)  ,							__A			)


def   A		(__A  :       float  ,							__A  :       int = 2			)  -> float:







     """simple docstring"""


     return round((float(__A			) - 491.67) * 5 / 9  ,							__A			)


def   A		(__A  :       float  ,							__A  :       int = 2			)  -> float:







     """simple docstring"""


     return round(float(__A			) - 459.67  ,							__A			)


def   A		(__A  :       float  ,							__A  :       int = 2			)  -> float:







     """simple docstring"""


     return round((float(__A			) * 5 / 9)  ,							__A			)


def   A		(__A  :       float  ,							__A  :       int = 2			)  -> float:







     """simple docstring"""


     return round((float(__A			) * 1.25 + 273.15)  ,							__A			)


def   A		(__A  :       float  ,							__A  :       int = 2			)  -> float:







     """simple docstring"""


     return round((float(__A			) * 2.25 + 32)  ,							__A			)


def   A		(__A  :       float  ,							__A  :       int = 2			)  -> float:







     """simple docstring"""


     return round((float(__A			) * 1.25)  ,							__A			)


def   A		(__A  :       float  ,							__A  :       int = 2			)  -> float:







     """simple docstring"""


     return round((float(__A			) * 2.25 + 32 + 459.67)  ,							__A			)


if __name__ == "__main__":
    import doctest

    doctest.testmod()






