


import os
import tempfile
import unittest
from typing import List

from transformers.models.esm.tokenization_esm import VOCAB_FILES_NAMES, EsmTokenizer
from transformers.testing_utils import require_tokenizers
from transformers.tokenization_utils import PreTrainedTokenizer
from transformers.tokenization_utils_base import PreTrainedTokenizerBase


@require_tokenizers
class 	a__						(    unittest.TestCase     ):
       lowerCamelCase__:  Any				     =  EsmTokenizer

       def     UpperCAmelCase( self   :  List[Any]     ):
            super().setUp()
            a_     :  Optional[int]				      =     tempfile.mkdtemp()
            # fmt: off
            a_     :  List[str]				      =     ["<cls>", "<pad>", "<eos>", "<unk>", "L", "A", "G", "V", "S", "E", "R", "T", "I", "D", "P", "K", "Q", "N", "F", "Y", "M", "H", "W", "C", "X", "B", "U", "Z", "O", ".", "-", "<null_1>", "<mask>"]  # noqa: E501
            # fmt: on
            a_     :  Optional[int]				      =     os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES["""vocab_file"""]     )
            with open(self.vocab_file   ,				"""w"""   ,				encoding="""utf-8"""     ) as vocab_writer:
                 vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]     )     )

       def     UpperCAmelCase( self   :  List[str]   ,				**lowerCamelCase_   :  int     ):
            return [self.get_tokenizer(**lowerCamelCase_     )]

       def     UpperCAmelCase( self   :  List[Any]   ,				**lowerCamelCase_   :  Tuple     ):
            return self.tokenizer_class.from_pretrained(self.tmpdirname   ,				**lowerCamelCase_     )

       def     UpperCAmelCase( self   :  List[str]     ):
            a_     :  Optional[int]				      =     self.tokenizer_class(self.vocab_file     )

            a_     :  Any				      =     tokenizer.tokenize("""LAGVS"""     )
            self.assertListEqual(lowerCamelCase_   ,				["""L""", """A""", """G""", """V""", """S"""]     )
            self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_     )   ,				[4, 5, 6, 7, 8]     )

       def     UpperCAmelCase( self   :  Dict     ):
            a_     :  Tuple				      =     self.tokenizer_class(self.vocab_file     )

            a_     :  Tuple				      =     """LAGVS"""
            self.assertListEqual(tokenizer.encode(lowerCamelCase_     )   ,				[0, 4, 5, 6, 7, 8, 2]     )

       def     UpperCAmelCase( self   :  Any     ):
            a_     :  List[str]				      =     self.tokenizer_class(self.vocab_file     )

            a_     :  int				      =     ["""LAGVS""", """WCB"""]
            a_     :  Any				      =     tokenizer(lowerCamelCase_   ,				padding=lowerCamelCase_     )["""input_ids"""]

            self.assertListEqual(lowerCamelCase_   ,				[[0, 4, 5, 6, 7, 8, 2], [0, 2_2, 2_3, 2_5, 2]]     )

       def     UpperCAmelCase( self   :  List[Any]     ):
            a_     :  Union[str, Any]				      =     self.tokenizer_class(self.vocab_file     )

            a_     :  Optional[Any]				      =     ["""LAGVS""", """WCB"""]
            a_     :  Dict				      =     tokenizer(lowerCamelCase_   ,				padding=lowerCamelCase_     )["""input_ids"""]

            self.assertListEqual(lowerCamelCase_   ,				[[0, 4, 5, 6, 7, 8, 2], [0, 2_2, 2_3, 2_5, 2, 1, 1]]     )

       def     UpperCAmelCase( self   :  Optional[Any]     ):
            a_     :  int				      =     self.get_tokenizers(fast=lowerCamelCase_     )
            for tokenizer in tokenizers:
                 with self.subTest(F'''{tokenizer.__class__.__name__}'''     ):
                      a_     :  Union[str, Any]				      =     """<unk>"""
                      a_     :  Tuple				      =     """<mask>"""

                      a_     :  Union[str, Any]				      =     tokenizer.tokenize(lowerCamelCase_     )
                      a_     :  Tuple				      =     tokenizer.tokenize(lowerCamelCase_     )

                      self.assertEqual(len(lowerCamelCase_     )   ,				1     )
                      self.assertEqual(len(lowerCamelCase_     )   ,				1     )
                      self.assertEqual(token_a[0]   ,				lowerCamelCase_     )
                      self.assertEqual(token_a[0]   ,				lowerCamelCase_     )











import logging
import os
import sys
import threading
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

from tqdm import auto as tqdm_lib


lowerCamelCase					    =	threading.Lock()
lowerCamelCase					    =	None

lowerCamelCase					    =	{
    '''debug''': logging.DEBUG,
    '''info''': logging.INFO,
    '''warning''': logging.WARNING,
    '''error''': logging.ERROR,
    '''critical''': logging.CRITICAL,
}

lowerCamelCase					    =	logging.WARNING

lowerCamelCase					    =	True
def 							UpperCAmelCase   ( ):


    '''simple docstring'''



    _lowercase     			=						os.getenv('DIFFUSERS_VERBOSITY',   _lowercase   )
    if env_level_str:
        if env_level_str in log_levels:
            return log_levels[env_level_str]
        else:
            logging.getLogger().warning(
                f'''Unknown option DIFFUSERS_VERBOSITY={env_level_str}, '''
                f'''has to be one of: { ", ".join(log_levels.keys()   ) }'''   )
    return _default_log_level
def 							UpperCAmelCase   ( ):


    '''simple docstring'''



    return __name__.split('.'   )[0]
def 							UpperCAmelCase   ( ):


    '''simple docstring'''



    return logging.getLogger(_get_library_name()   )
def 							UpperCAmelCase   ( ):


    '''simple docstring'''



    global _default_handler

    with _lock:
        if _default_handler:
            # This library has already configured the library root logger.
            return
        _lowercase     			=						logging.StreamHandler()  # Set sys.stderr as stream.
        _lowercase     			=						sys.stderr.flush

        # Apply our default configuration to the library root logger.
        _lowercase     			=						_get_library_root_logger()
        library_root_logger.addHandler(_default_handler   )
        library_root_logger.setLevel(_get_default_logging_level()   )
        _lowercase     			=						False
def 							UpperCAmelCase   ( ):


    '''simple docstring'''



    global _default_handler

    with _lock:
        if not _default_handler:
            return

        _lowercase     			=						_get_library_root_logger()
        library_root_logger.removeHandler(_default_handler   )
        library_root_logger.setLevel(logging.NOTSET   )
        _lowercase     			=						None
def 							UpperCAmelCase   ( ):


    '''simple docstring'''



    return log_levels
def 							UpperCAmelCase   ( _lowercase = None   ):


    '''simple docstring'''



    if name is None:
        _lowercase     			=						_get_library_name()

    _configure_library_root_logger()
    return logging.getLogger(_lowercase   )
def 							UpperCAmelCase   ( ):


    '''simple docstring'''



    _configure_library_root_logger()
    return _get_library_root_logger().getEffectiveLevel()
def 							UpperCAmelCase   ( _lowercase   ):


    '''simple docstring'''



    _configure_library_root_logger()
    _get_library_root_logger().setLevel(_lowercase   )
def 							UpperCAmelCase   ( ):


    '''simple docstring'''



    return set_verbosity(_lowercase   )
def 							UpperCAmelCase   ( ):


    '''simple docstring'''



    return set_verbosity(_lowercase   )
def 							UpperCAmelCase   ( ):


    '''simple docstring'''



    return set_verbosity(_lowercase   )
def 							UpperCAmelCase   ( ):


    '''simple docstring'''



    return set_verbosity(_lowercase   )
def 							UpperCAmelCase   ( ):


    '''simple docstring'''



    _configure_library_root_logger()

    assert _default_handler is not None
    _get_library_root_logger().removeHandler(_default_handler   )
def 							UpperCAmelCase   ( ):


    '''simple docstring'''



    _configure_library_root_logger()

    assert _default_handler is not None
    _get_library_root_logger().addHandler(_default_handler   )
def 							UpperCAmelCase   ( _lowercase   ):


    '''simple docstring'''



    _configure_library_root_logger()

    assert handler is not None
    _get_library_root_logger().addHandler(_lowercase   )
def 							UpperCAmelCase   ( _lowercase   ):


    '''simple docstring'''



    _configure_library_root_logger()

    assert handler is not None and handler not in _get_library_root_logger().handlers
    _get_library_root_logger().removeHandler(_lowercase   )
def 							UpperCAmelCase   ( ):


    '''simple docstring'''



    _configure_library_root_logger()
    _lowercase     			=						False
def 							UpperCAmelCase   ( ):


    '''simple docstring'''



    _configure_library_root_logger()
    _lowercase     			=						True
def 							UpperCAmelCase   ( ):


    '''simple docstring'''



    _lowercase     			=						_get_library_root_logger().handlers

    for handler in handlers:
        _lowercase     			=						logging.Formatter('[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s'   )
        handler.setFormatter(_lowercase   )
def 							UpperCAmelCase   ( ):


    '''simple docstring'''



    _lowercase     			=						_get_library_root_logger().handlers

    for handler in handlers:
        handler.setFormatter(_lowercase   )
def 							UpperCAmelCase   ( self,   *_lowercase,   **_lowercase   ):


    '''simple docstring'''



    _lowercase     			=						os.getenv('DIFFUSERS_NO_ADVISORY_WARNINGS',   _lowercase   )
    if no_advisory_warnings:
        return
    self.warning(*_lowercase,   **_lowercase   )


lowerCamelCase					    =	warning_advice

class        snake_case_				:







   """simple docstring"""



   def __init__(     self:						int ,							*snake_case__:						str ,							**snake_case__:						Any						)	->		Any:  # pylint: disable=unused-argument


       """simple docstring"""



       _lowercase     			=						args[0] if args else None

   def __iter__(     self:						Union[str, Any]						)	->		str:


       """simple docstring"""



       return iter(self._iterator						)

   def __getattr__(     self:						int ,							snake_case__:						Any						)	->		List[Any]:


       """simple docstring"""



       def empty_fn(*snake_case__:						Optional[int] ,							**snake_case__:						Union[str, Any]						):  # pylint: disable=unused-argument
           return

       return empty_fn

   def __enter__(     self:						List[Any]						)	->		Tuple:


       """simple docstring"""



       return self

   def __exit__(     self:						List[str] ,							snake_case__:						Optional[Any] ,							snake_case__:						Dict ,							snake_case__:						List[str]						)	->		Optional[Any]:


       """simple docstring"""



       return






class        snake_case_				:







   """simple docstring"""



   def __call__(     self:						List[str] ,							*snake_case__:						Tuple ,							**snake_case__:						Union[str, Any]						)	->		List[str]:


       """simple docstring"""



       if _tqdm_active:
           return tqdm_lib.tqdm(*snake_case__ ,							**snake_case__						)
       else:
           return EmptyTqdm(*snake_case__ ,							**snake_case__						)

   def       SCREAMING_SNAKE_CASE__					(     self:						Tuple ,							*snake_case__:						Union[str, Any] ,							**snake_case__:						str						)	->		Optional[int]:


       """simple docstring"""



       _lowercase     			=						None
       if _tqdm_active:
           return tqdm_lib.tqdm.set_lock(*snake_case__ ,							**snake_case__						)

   def       SCREAMING_SNAKE_CASE__					(     self:						int						)	->		int:


       """simple docstring"""



       if _tqdm_active:
           return tqdm_lib.tqdm.get_lock()






lowerCamelCase					    =	_tqdm_cls()
def 							UpperCAmelCase   ( ):


    '''simple docstring'''



    global _tqdm_active
    return bool(_tqdm_active   )
def 							UpperCAmelCase   ( ):


    '''simple docstring'''



    global _tqdm_active
    _lowercase     			=						True
def 							UpperCAmelCase   ( ):


    '''simple docstring'''



    global _tqdm_active
    _lowercase     			=						False
