





'''simple docstring'''






import os
import unittest

from huggingface_hub.utils import are_progress_bars_disabled

import transformers.models.bart.tokenization_bart
from transformers import logging
from transformers.testing_utils import CaptureLogger, mockenv, mockenv_context
from transformers.utils.logging import disable_progress_bar, enable_progress_bar




class  UpperCAmelCase     (			unittest.TestCase   ):




      def 				lowercase__    (			self			: List[Any]  )				->      Optional[Any]:
            _lowerCAmelCase							     =							logging.get_logger()

            # the current default level is logging.WARNING
            _lowerCAmelCase							     =							logging.get_verbosity()

            logging.set_verbosity_error()
            self.assertEqual(logger.getEffectiveLevel()			,			logging.get_verbosity()  )

            logging.set_verbosity_warning()
            self.assertEqual(logger.getEffectiveLevel()			,			logging.get_verbosity()  )

            logging.set_verbosity_info()
            self.assertEqual(logger.getEffectiveLevel()			,			logging.get_verbosity()  )

            logging.set_verbosity_debug()
            self.assertEqual(logger.getEffectiveLevel()			,			logging.get_verbosity()  )

            # restore to the original level
            logging.set_verbosity(__snake_case  )




      def 				lowercase__    (			self			: int  )				->      str:
            _lowerCAmelCase							     =							logging.get_verbosity()

            _lowerCAmelCase							     =							logging.get_logger("""transformers.models.bart.tokenization_bart"""  )
            _lowerCAmelCase							     =							"""Testing 1, 2, 3"""

            # should be able to log warnings (if default settings weren't overridden by `pytest --log-level-all`)
            if level_origin <= logging.WARNING:
                  with CaptureLogger(__snake_case  ) as cl:
                        logger.warning(__snake_case  )
                  self.assertEqual(cl.out			,			msg + """\n"""  )

            # this is setting the level for all of `transformers.*` loggers
            logging.set_verbosity_error()

            # should not be able to log warnings
            with CaptureLogger(__snake_case  ) as cl:
                  logger.warning(__snake_case  )
            self.assertEqual(cl.out			,			""""""  )

            # should be able to log warnings again
            logging.set_verbosity_warning()
            with CaptureLogger(__snake_case  ) as cl:
                  logger.warning(__snake_case  )
            self.assertEqual(cl.out			,			msg + """\n"""  )

            # restore to the original level
            logging.set_verbosity(__snake_case  )




      @mockenv(TRANSFORMERS_VERBOSITY="""error"""  )
      def 				lowercase__    (			self			: Tuple  )				->      Tuple:
            # reset for the env var to take effect, next time some logger call is made
            transformers.utils.logging._reset_library_root_logger()
            # this action activates the env var
            _lowerCAmelCase							     =							logging.get_logger("""transformers.models.bart.tokenization_bart"""  )

            _lowerCAmelCase							     =							os.getenv("""TRANSFORMERS_VERBOSITY"""			,			__snake_case  )
            _lowerCAmelCase							     =							logging.log_levels[env_level_str]

            _lowerCAmelCase							     =							logging.get_verbosity()
            self.assertEqual(
                __snake_case			,			__snake_case			,			f"TRANSFORMERS_VERBOSITY={env_level_str}/{env_level}, but internal verbosity is {current_level}"			,			)

            # restore to the original level
            _lowerCAmelCase							     =							""""""
            transformers.utils.logging._reset_library_root_logger()




      @mockenv(TRANSFORMERS_VERBOSITY="""super-error"""  )
      def 				lowercase__    (			self			: Optional[int]  )				->      Any:
            # reset for the env var to take effect, next time some logger call is made
            transformers.utils.logging._reset_library_root_logger()
            _lowerCAmelCase							     =							logging.logging.getLogger()
            with CaptureLogger(__snake_case  ) as cl:
                  # this action activates the env var
                  logging.get_logger("""transformers.models.bart.tokenization_bart"""  )
            self.assertIn("""Unknown option TRANSFORMERS_VERBOSITY=super-error"""			,			cl.out  )

            # no need to restore as nothing was changed




      def 				lowercase__    (			self			: Dict  )				->      Any:
            # testing `logger.warning_advice()`
            transformers.utils.logging._reset_library_root_logger()

            _lowerCAmelCase							     =							logging.get_logger("""transformers.models.bart.tokenization_bart"""  )
            _lowerCAmelCase							     =							"""Testing 1, 2, 3"""

            with mockenv_context(TRANSFORMERS_NO_ADVISORY_WARNINGS="""1"""  ):
                  # nothing should be logged as env var disables this method
                  with CaptureLogger(__snake_case  ) as cl:
                        logger.warning_advice(__snake_case  )
                  self.assertEqual(cl.out			,			""""""  )

            with mockenv_context(TRANSFORMERS_NO_ADVISORY_WARNINGS=""""""  ):
                  # should log normally as TRANSFORMERS_NO_ADVISORY_WARNINGS is unset
                  with CaptureLogger(__snake_case  ) as cl:
                        logger.warning_advice(__snake_case  )
                  self.assertEqual(cl.out			,			msg + """\n"""  )




def    UpperCamelCase__      (		):





      """simple docstring"""


      disable_progress_bar()
      assert are_progress_bars_disabled()

      enable_progress_bar()
      assert not are_progress_bars_disabled()









import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


lowerCAmelCase_       	=	{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCAmelCase_       	=	{
    '''vocab_file''': {
        '''unc-nlp/lxmert-base-uncased''': '''https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''unc-nlp/lxmert-base-uncased''': (
            '''https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json'''
        ),
    },
}

lowerCAmelCase_       	=	{
    '''unc-nlp/lxmert-base-uncased''': 5_12,
}

lowerCAmelCase_       	=	{
    '''unc-nlp/lxmert-base-uncased''': {'''do_lower_case''': True},
}





class 	snake_case_   (		__A					):







			'''simple docstring'''

			SCREAMING_SNAKE_CASE     :   Optional[Any]									=				VOCAB_FILES_NAMES
			SCREAMING_SNAKE_CASE     :   Any									=				PRETRAINED_VOCAB_FILES_MAP
			SCREAMING_SNAKE_CASE     :   List[Any]									=				PRETRAINED_INIT_CONFIGURATION
			SCREAMING_SNAKE_CASE     :   Optional[int]									=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			SCREAMING_SNAKE_CASE     :   Any									=				LxmertTokenizer







			def __init__(     self :				Union[str, Any]							,   _UpperCamelCase :				int=None							,   _UpperCamelCase :				Optional[Any]=None							,   _UpperCamelCase :				Dict=True							,   _UpperCamelCase :				Any="[UNK]"							,   _UpperCamelCase :				Tuple="[SEP]"							,   _UpperCamelCase :				List[Any]="[PAD]"							,   _UpperCamelCase :				Union[str, Any]="[CLS]"							,   _UpperCamelCase :				str="[MASK]"							,   _UpperCamelCase :				List[str]=True							,   _UpperCamelCase :				List[str]=None							,   **_UpperCamelCase :				List[str]							,   )      ->Any:
									super().__init__(
									    _UpperCamelCase							,   tokenizer_file=_UpperCamelCase							,   do_lower_case=_UpperCamelCase							,   unk_token=_UpperCamelCase							,   sep_token=_UpperCamelCase							,   pad_token=_UpperCamelCase							,   cls_token=_UpperCamelCase							,   mask_token=_UpperCamelCase							,   tokenize_chinese_chars=_UpperCamelCase							,   strip_accents=_UpperCamelCase							,   **_UpperCamelCase							,   )

									snake_case_   =	json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
									if (
									    normalizer_state.get('''lowercase'''							,   _UpperCamelCase							) != do_lower_case
									    or normalizer_state.get('''strip_accents'''							,   _UpperCamelCase							) != strip_accents
									    or normalizer_state.get('''handle_chinese_chars'''							,   _UpperCamelCase							) != tokenize_chinese_chars
									):
															snake_case_   =	getattr(_UpperCamelCase							,   normalizer_state.pop('''type'''							)							)
															snake_case_   =	do_lower_case
															snake_case_   =	strip_accents
															snake_case_   =	tokenize_chinese_chars
															snake_case_   =	normalizer_class(**_UpperCamelCase							)

									snake_case_   =	do_lower_case







			def 				snake_case__(     self :				Optional[int]							,   _UpperCamelCase :				List[Any]							,   _UpperCamelCase :				List[str]=None							)      ->List[Any]:
									snake_case_   =	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
															output += token_ids_a + [self.sep_token_id]

									return output







			def 				snake_case__(     self :				int							,   _UpperCamelCase :				List[int]							,   _UpperCamelCase :				Optional[List[int]] = None							)      ->List[int]:
									snake_case_   =	[self.sep_token_id]
									snake_case_   =	[self.cls_token_id]
									if token_ids_a is None:
															return len(cls + token_ids_a + sep							) * [0]
									return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







			def 				snake_case__(     self :				Any							,   _UpperCamelCase :				str							,   _UpperCamelCase :				Optional[str] = None							)      ->Tuple[str]:
									snake_case_   =	self._tokenizer.model.save(_UpperCamelCase							,   name=_UpperCamelCase							)
									return tuple(_UpperCamelCase							)