


"""simple docstring"""







import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
       if is_torch_available():
              import torch
       if is_tf_available():
              import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


lowerCamelCase_  :  str     		=    logging.get_logger(__name__)

lowerCamelCase_  :  int     		=    {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

lowerCamelCase_  :  Optional[Any]     		=    {
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
}

lowerCamelCase_  :  List[str]     		=    {
    """Salesforce/codegen-350M-mono""": 2_0_4_8,
}
@lru_cache()
def _A		(		):







    """simple docstring"""
    a											=(
        list(range(ord('''!'''						)			,       ord('''~'''						) + 1						)						) + list(range(ord('''¡'''						)			,       ord('''¬'''						) + 1						)						) + list(range(ord('''®'''						)			,       ord('''ÿ'''						) + 1						)						)
    )
    a											=bs[:]
    a											=0
    for b in range(2**8						):
        if b not in bs:
            bs.append(lowercase						)
            cs.append(2**8 + n						)
            n += 1
    a											=[chr(lowercase						) for n in cs]
    return dict(zip(lowercase			,       lowercase						)						)
def _A		(		lowercase						):







    """simple docstring"""
    a											=set()
    a											=word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)						)
        a											=char
    return pairs






class  __A							(			_SCREAMING_SNAKE_CASE     ):






 """simple docstring"""




 __lowerCAmelCase					   =    VOCAB_FILES_NAMES
 __lowerCAmelCase					   =    PRETRAINED_VOCAB_FILES_MAP
 __lowerCAmelCase					   =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowerCAmelCase					   =    ["input_ids", "attention_mask"]


 def __init__(						self					,       __A					,       __A					,       __A="replace"					,       __A="<|endoftext|>"					,       __A="<|endoftext|>"					,       __A="<|endoftext|>"					,       __A=None					,       __A=False					,       __A=False					,       **__A					,       )   ->	int:
     a											=AddedToken(__A					,       lstrip=__A					,       rstrip=__A		) if isinstance(__A					,       __A		) else bos_token
     a											=AddedToken(__A					,       lstrip=__A					,       rstrip=__A		) if isinstance(__A					,       __A		) else eos_token
     a											=AddedToken(__A					,       lstrip=__A					,       rstrip=__A		) if isinstance(__A					,       __A		) else unk_token
     a											=AddedToken(__A					,       lstrip=__A					,       rstrip=__A		) if isinstance(__A					,       __A		) else pad_token
     super().__init__(
         errors=__A					,       unk_token=__A					,       bos_token=__A					,       eos_token=__A					,       pad_token=__A					,       add_prefix_space=__A					,       add_bos_token=__A					,       **__A					,       )
     a											=add_bos_token

     with open(__A					,       encoding='''utf-8'''		) as vocab_handle:
         a											=json.load(__A		)
     a											={v: k for k, v in self.encoder.items()}
     a											=errors  # how to handle errors in decoding
     a											=bytes_to_unicode()
     a											={v: k for k, v in self.byte_encoder.items()}
     with open(__A					,       encoding='''utf-8'''		) as merges_handle:
         a											=merges_handle.read().split('''\n'''		)[1:-1]
     a											=[tuple(merge.split()		) for merge in bpe_merges]
     a											=dict(zip(__A					,       range(len(__A		)		)		)		)
     a											={}
     a											=add_prefix_space

     # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
     a											=re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''		)


 @property
 def  SCREAMING_SNAKE_CASE				(						self		)   ->	List[str]:
     return len(self.encoder		)


 def  SCREAMING_SNAKE_CASE				(						self		)   ->	Optional[Any]:
     return dict(self.encoder					,       **self.added_tokens_encoder		)


 def  SCREAMING_SNAKE_CASE				(						self					,       __A		)   ->	List[str]:
     if token in self.cache:
         return self.cache[token]
     a											=tuple(__A		)
     a											=get_pairs(__A		)

     if not pairs:
         return token

     while True:
         a											=min(__A					,       key=lambda __A		: self.bpe_ranks.get(__A					,       float('''inf'''		)		)		)
         if bigram not in self.bpe_ranks:
             break
         a						, a											=bigram
         a											=[]
         a											=0
         while i < len(__A		):
             try:
                 a											=word.index(__A					,       __A		)
             except ValueError:
                 new_word.extend(word[i:]		)
                 break
             else:
                 new_word.extend(word[i:j]		)
                 a											=j

             if word[i] == first and i < len(__A		) - 1 and word[i + 1] == second:
                 new_word.append(first + second		)
                 i += 2
             else:
                 new_word.append(word[i]		)
                 i += 1
         a											=tuple(__A		)
         a											=new_word
         if len(__A		) == 1:
             break
         else:
             a											=get_pairs(__A		)
     a											=''' '''.join(__A		)
     a											=word
     return word


 def  SCREAMING_SNAKE_CASE				(						self					,       __A					,       __A=None		)   ->	Optional[Any]:
     if self.add_bos_token:
         a											=[self.bos_token_id]
     else:
         a											=[]

     a											=bos_token_ids + token_ids_a

     if token_ids_a is None:
         return output

     return output + bos_token_ids + token_ids_a


 def  SCREAMING_SNAKE_CASE				(						self					,       __A		)   ->	List[str]:
     a											=[]
     for token in re.findall(self.pat					,       __A		):
         a											=''''''.join(
             self.byte_encoder[b] for b in token.encode('''utf-8'''		)		)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
         bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__A		).split(''' '''		)		)
     return bpe_tokens


 def  SCREAMING_SNAKE_CASE				(						self					,       __A		)   ->	Optional[int]:
     return self.encoder.get(__A					,       self.encoder.get(self.unk_token		)		)


 def  SCREAMING_SNAKE_CASE				(						self					,       __A		)   ->	Any:
     return self.decoder.get(__A		)


 def  SCREAMING_SNAKE_CASE				(						self					,       __A		)   ->	List[Any]:
     a											=''''''.join(__A		)
     a											=bytearray([self.byte_decoder[c] for c in text]		).decode('''utf-8'''					,       errors=self.errors		)
     return text


 def  SCREAMING_SNAKE_CASE				(						self					,       __A					,       __A = None		)   ->	Tuple[str]:
     if not os.path.isdir(__A		):
         logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''		)
         return
     a											=os.path.join(
         __A					,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)
     a											=os.path.join(
         __A					,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']		)

     with open(__A					,       '''w'''					,       encoding='''utf-8'''		) as f:
         f.write(json.dumps(self.encoder					,       indent=2					,       sort_keys=__A					,       ensure_ascii=__A		) + '''\n'''		)

     a											=0
     with open(__A					,       '''w'''					,       encoding='''utf-8'''		) as writer:
         writer.write('''#version: 0.2\n'''		)
         for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,       key=lambda __A		: kv[1]		):
             if index != token_index:
                 logger.warning(
                     f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                     ''' Please check that the tokenizer is not corrupted!'''		)
                 a											=token_index
             writer.write(''' '''.join(__A		) + '''\n'''		)
             index += 1

     return vocab_file, merge_file


 def  SCREAMING_SNAKE_CASE				(						self					,       __A					,       __A=False					,       **__A		)   ->	Optional[int]:
     a											=kwargs.pop('''add_prefix_space'''					,       self.add_prefix_space		)
     if is_split_into_words or add_prefix_space:
         a											=''' ''' + text
     return (text, kwargs)


 def  SCREAMING_SNAKE_CASE				(						self					,       __A					,       __A = False					,       __A = None					,       __A = None					,       **__A					,       )   ->	str:
     a											=super()._decode(
         token_ids=__A					,       skip_special_tokens=__A					,       clean_up_tokenization_spaces=__A					,       **__A					,       )

     if truncate_before_pattern is not None and len(__A		) > 0:
         a											=self.truncate(__A					,       __A		)

     return decoded_text





 def  SCREAMING_SNAKE_CASE				(						self					,       __A					,       __A		)   ->	List[Any]:
     def find_re(__A					,       __A					,       __A		):
         a											=pattern.search(__A					,       __A		)
         return m.start() if m else -1

     a											=[re.compile(__A					,       re.MULTILINE		) for pattern in truncate_before_pattern]

     a											=list(re.finditer('''^print'''					,       __A					,       re.MULTILINE		)		)

     if len(__A		) > 1:
         a											=completion[: prints[1].start()]

     a											=list(re.finditer('''^def'''					,       __A					,       re.MULTILINE		)		)

     if len(__A		) > 1:
         a											=completion[: defs[1].start()]

     a											=0

     a											=[
         pos for pos in [find_re(__A					,       __A					,       __A		) for terminal in terminals] if pos != -1
     ]

     if len(__A		) > 0:
         return completion[: min(__A		)]
     else:
         return completion



"""simple docstring"""







import pytest

from datasets.utils.version import Version
@pytest.mark.parametrize(
    '''other, expected_equality'''			,       [
        (Version('''1.0.0'''						), True),
        ('''1.0.0''', True),
        (Version('''2.0.0'''						), False),
        ('''2.0.0''', False),
        ('''1''', False),
        ('''a''', False),
        (1, False),
        (None, False),
    ]			,       )
def _A		(		lowercase			,       lowercase						):







    """simple docstring"""
    a											=Version('''1.0.0'''						)
    assert (version == other) is expected_equality
    assert (version != other) is not expected_equality
    assert (hash(lowercase						) == hash(lowercase						)) is expected_equality