





'''simple docstring'''

import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin
class 				__lowerCAmelCase     (    __snake_case						,						unittest.TestCase ):






       """simple docstring"""
       _snake_case	:     Union[str, Any]										=    CanineTokenizer
       _snake_case	:     Optional[Any]										=    False





       def snake_case__						(							self   :		int     )    ->     str:

            '''simple docstring'''




            super().setUp()
            _UpperCamelCase				    =       CanineTokenizer()
            tokenizer.save_pretrained(self.tmpdirname     )





       @cached_property
       def snake_case__						(							self   :		str     )    ->     Tuple:

            '''simple docstring'''




            return CanineTokenizer.from_pretrained('''google/canine-s'''     )





       def snake_case__						(							self   :		Dict       , **lowerCAmelCase__   :		Tuple     )    ->     Optional[Any]:

            '''simple docstring'''




            _UpperCamelCase				    =       self.tokenizer_class.from_pretrained(self.tmpdirname       , **A_     )
            _UpperCamelCase				    =       1024
            return tokenizer





       @require_torch
       def snake_case__						(							self   :		Optional[Any]     )    ->     str:

            '''simple docstring'''




            _UpperCamelCase				    =       self.canine_tokenizer
            _UpperCamelCase				    =       ["Life is like a box of chocolates.", "You never know what you're gonna get."]
            # fmt: off
            _UpperCamelCase				    =       [57344, 76, 105, 102, 101, 32, 105, 115, 32, 108, 105, 107, 101, 32, 97, 32, 98, 111, 120, 32, 111, 102, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 115, 46, 57345, 0, 0, 0, 0]
            # fmt: on
            _UpperCamelCase				    =       tokenizer(A_       , padding=A_       , return_tensors='''pt'''     )
            self.assertIsInstance(A_       , A_     )

            _UpperCamelCase				    =       list(batch.input_ids.numpy()[0]     )

            self.assertListEqual(A_       , A_     )

            self.assertEqual((2, 39)       , batch.input_ids.shape     )
            self.assertEqual((2, 39)       , batch.attention_mask.shape     )





       @require_torch
       def snake_case__						(							self   :		Dict     )    ->     Dict:

            '''simple docstring'''




            _UpperCamelCase				    =       self.canine_tokenizer
            _UpperCamelCase				    =       ["Once there was a man.", "He wrote a test in HuggingFace Tranformers."]
            _UpperCamelCase				    =       tokenizer(A_       , padding=A_       , return_tensors='''pt'''     )
            # check if input_ids, attention_mask and token_type_ids are returned
            self.assertIn('''input_ids'''       , A_     )
            self.assertIn('''attention_mask'''       , A_     )
            self.assertIn('''token_type_ids'''       , A_     )





       @require_torch
       def snake_case__						(							self   :		int     )    ->     Optional[Any]:

            '''simple docstring'''




            _UpperCamelCase				    =       self.canine_tokenizer
            _UpperCamelCase				    =       [
                "What's the weater?",
                "It's about 25 degrees.",
            ]
            _UpperCamelCase				    =       tokenizer(
                text_target=A_       , max_length=32       , padding='''max_length'''       , truncation=A_       , return_tensors='''pt'''     )
            self.assertEqual(32       , targets['''input_ids'''].shape[1]     )





       def snake_case__						(							self   :		Union[str, Any]     )    ->     int:

            '''simple docstring'''




            _UpperCamelCase				    =       self.get_tokenizers()
            for tokenizer in tokenizers:
                 with self.subTest(f"""{tokenizer.__class__.__name__}"""     ):
                      self.assertNotEqual(tokenizer.model_max_length       , 42     )

                 # Now let's start the test
            _UpperCamelCase				    =       self.get_tokenizers()
            for tokenizer in tokenizers:
                 with self.subTest(f"""{tokenizer.__class__.__name__}"""     ):
                      # Isolate this from the other tests because we save additional tokens/etc
                      _UpperCamelCase				    =       tempfile.mkdtemp()

                      _UpperCamelCase				    =       " He is very happy, UNwant\u00E9d,running"
                      _UpperCamelCase				    =       tokenizer.encode(A_       , add_special_tokens=A_     )
                      tokenizer.save_pretrained(A_     )

                      _UpperCamelCase				    =       tokenizer.__class__.from_pretrained(A_     )
                      _UpperCamelCase				    =       after_tokenizer.encode(A_       , add_special_tokens=A_     )
                      self.assertListEqual(A_       , A_     )

                      shutil.rmtree(A_     )

            _UpperCamelCase				    =       self.get_tokenizers(model_max_length=42     )
            for tokenizer in tokenizers:
                 with self.subTest(f"""{tokenizer.__class__.__name__}"""     ):
                      # Isolate this from the other tests because we save additional tokens/etc
                      _UpperCamelCase				    =       tempfile.mkdtemp()

                      _UpperCamelCase				    =       " He is very happy, UNwant\u00E9d,running"

                      _UpperCamelCase				    =       tokenizer.additional_special_tokens

                      # We can add a new special token for Canine as follows:
                      _UpperCamelCase				    =       chr(0Xe0_07     )
                      additional_special_tokens.append(A_     )
                      tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}     )
                      _UpperCamelCase				    =       tokenizer.encode(A_       , add_special_tokens=A_     )
                      tokenizer.save_pretrained(A_     )

                      _UpperCamelCase				    =       tokenizer.__class__.from_pretrained(A_     )
                      _UpperCamelCase				    =       after_tokenizer.encode(A_       , add_special_tokens=A_     )
                      self.assertListEqual(A_       , A_     )
                      self.assertIn(A_       , after_tokenizer.additional_special_tokens     )
                      self.assertEqual(after_tokenizer.model_max_length       , 42     )

                      _UpperCamelCase				    =       tokenizer.__class__.from_pretrained(A_       , model_max_length=43     )
                      self.assertEqual(tokenizer.model_max_length       , 43     )

                      shutil.rmtree(A_     )





       def snake_case__						(							self   :		int     )    ->     List[Any]:

            '''simple docstring'''




            _UpperCamelCase				    =       self.get_tokenizers(do_lower_case=A_     )
            for tokenizer in tokenizers:
                 with self.subTest(f"""{tokenizer.__class__.__name__}"""     ):
                      _UpperCamelCase				    =       self.get_clean_sequence(A_     )

                      # a special token for Canine can be defined as follows:
                      _UpperCamelCase				    =       0Xe0_05
                      _UpperCamelCase				    =       chr(A_     )

                      tokenizer.add_special_tokens({'''cls_token''': special_token}     )
                      _UpperCamelCase				    =       tokenizer.encode(A_       , add_special_tokens=A_     )
                      self.assertEqual(len(A_     )       , 1     )

                      _UpperCamelCase				    =       tokenizer.decode(ids + encoded_special_token       , clean_up_tokenization_spaces=A_     )
                      _UpperCamelCase				    =       tokenizer.encode(A_       , add_special_tokens=A_     )

                      _UpperCamelCase				    =       tokenizer.encode(A_       , add_special_tokens=A_     )
                      _UpperCamelCase				    =       tokenizer.encode(A_       , add_special_tokens=A_     )
                      self.assertEqual(A_       , input_encoded + special_token_id     )

                      _UpperCamelCase				    =       tokenizer.decode(A_       , skip_special_tokens=A_     )
                      self.assertTrue(special_token not in decoded     )





       def snake_case__						(							self   :		List[str]     )    ->     Any:

            '''simple docstring'''




            _UpperCamelCase				    =       self.get_tokenizers(do_lower_case=A_     )
            for tokenizer in tokenizers:
                 with self.subTest(f"""{tokenizer.__class__.__name__}"""     ):
                      _UpperCamelCase				    =       chr(0Xe0_05     )
                      _UpperCamelCase				    =       chr(0Xe0_06     )

                      # `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
                      tokenizer.add_tokens([SPECIAL_TOKEN_1]       , special_tokens=A_     )
                      # `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
                      # which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
                      tokenizer.add_special_tokens({'''additional_special_tokens''': [SPECIAL_TOKEN_2]}     )

                      _UpperCamelCase				    =       tokenizer.tokenize(A_     )
                      _UpperCamelCase				    =       tokenizer.tokenize(A_     )

                      self.assertEqual(len(A_     )       , 1     )
                      self.assertEqual(len(A_     )       , 1     )
                      self.assertEqual(token_a[0]       , A_     )
                      self.assertEqual(token_a[0]       , A_     )





       @require_tokenizers
       def snake_case__						(							self   :		Any     )    ->     int:

            '''simple docstring'''




            _UpperCamelCase				    =       self.get_tokenizers(do_lower_case=A_     )
            for tokenizer in tokenizers:
                 with self.subTest(f"""{tokenizer.__class__.__name__}"""     ):
                      # a special token for Canine can be defined as follows:
                      _UpperCamelCase				    =       0Xe0_06
                      _UpperCamelCase				    =       chr(A_     )

                      _UpperCamelCase				    =       AddedToken(A_       , lstrip=A_     )
                      tokenizer.add_special_tokens({'''additional_special_tokens''': [new_token]}     )

                      with tempfile.TemporaryDirectory() as tmp_dir_name:
                           tokenizer.save_pretrained(A_     )
                           tokenizer.from_pretrained(A_     )





       def snake_case__						(							self   :		List[str]     )    ->     Any:

            '''simple docstring'''




            _UpperCamelCase				    =       []
            if self.test_slow_tokenizer:
                 tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())     )

            if self.test_rust_tokenizer:
                 tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())     )

            for tokenizer_class, tokenizer_utils in tokenizer_list:
                 with tempfile.TemporaryDirectory() as tmp_dir:
                      tokenizer_utils.save_pretrained(A_     )

                      with open(os.path.join(A_       , '''special_tokens_map.json'''     )       , encoding='''utf-8'''     ) as json_file:
                           _UpperCamelCase				    =       json.load(A_     )

                      with open(os.path.join(A_       , '''tokenizer_config.json'''     )       , encoding='''utf-8'''     ) as json_file:
                           _UpperCamelCase				    =       json.load(A_     )

                      # a special token for Canine can be defined as follows:
                      _UpperCamelCase				    =       0Xe0_06
                      _UpperCamelCase				    =       chr(A_     )

                      _UpperCamelCase				    =       [new_token_a]
                      _UpperCamelCase				    =       [new_token_a]

                      with open(os.path.join(A_       , '''special_tokens_map.json'''     )       , '''w'''       , encoding='''utf-8'''     ) as outfile:
                           json.dump(A_       , A_     )
                      with open(os.path.join(A_       , '''tokenizer_config.json'''     )       , '''w'''       , encoding='''utf-8'''     ) as outfile:
                           json.dump(A_       , A_     )

                      # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                      # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                      # "special_tokens_map.json" files
                      _UpperCamelCase				    =       tokenizer_class.from_pretrained(A_       , extra_ids=0     )
                      self.assertIn(A_       , tokenizer_without_change_in_init.additional_special_tokens     )
                      # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                      self.assertEqual(
                          [new_token_a]       , tokenizer_without_change_in_init.convert_ids_to_tokens(
                              tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]     )     )       , )

                      _UpperCamelCase				    =       0Xe0_07
                      _UpperCamelCase				    =       chr(A_     )
                      # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                      _UpperCamelCase				    =       [AddedToken(A_       , lstrip=A_     )]
                      _UpperCamelCase				    =       tokenizer_class.from_pretrained(
                          A_       , additional_special_tokens=A_       , extra_ids=0     )

                      self.assertIn(A_       , tokenizer.additional_special_tokens     )
                      # self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
                      self.assertEqual(
                          [new_token_a]       , tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]     )     )     )





       @require_tokenizers
       def snake_case__						(							self   :		str     )    ->     Tuple:

            '''simple docstring'''




            _UpperCamelCase				    =       self.get_tokenizers(do_lower_case=A_     )
            for tokenizer in tokenizers:
                 with self.subTest(f"""{tokenizer.__class__.__name__}"""     ):
                      _UpperCamelCase				    =       "hello world"
                      if self.space_between_special_tokens:
                           _UpperCamelCase				    =       "[CLS] hello world [SEP]"
                      else:
                           _UpperCamelCase				    =       input
                      _UpperCamelCase				    =       tokenizer.encode(A_       , add_special_tokens=A_     )
                      _UpperCamelCase				    =       tokenizer.decode(A_       , spaces_between_special_tokens=self.space_between_special_tokens     )
                      self.assertIn(A_       , [output, output.lower()]     )





       def snake_case__						(							self   :		str     )    ->     Any:

            '''simple docstring'''




            _UpperCamelCase				    =       self.get_tokenizers()
            for tokenizer in tokenizers:
                 with self.subTest(f"""{tokenizer.__class__.__name__}"""     ):
                      _UpperCamelCase				    =       [
                          "bos_token",
                          "eos_token",
                          "unk_token",
                          "sep_token",
                          "pad_token",
                          "cls_token",
                          "mask_token",
                      ]

            _UpperCamelCase				    =       "a"
            _UpperCamelCase				    =       ord(A_     )

            for attr in attributes_list:
                 setattr(A_       , attr + '''_id'''       , A_     )
                 self.assertEqual(getattr(A_       , A_     )       , A_     )
                 self.assertEqual(getattr(A_       , attr + '''_id'''     )       , A_     )

                 setattr(A_       , attr + '''_id'''       , A_     )
                 self.assertEqual(getattr(A_       , A_     )       , A_     )
                 self.assertEqual(getattr(A_       , attr + '''_id'''     )       , A_     )

            setattr(A_       , '''additional_special_tokens_ids'''       , []     )
            self.assertListEqual(getattr(A_       , '''additional_special_tokens'''     )       , []     )
            self.assertListEqual(getattr(A_       , '''additional_special_tokens_ids'''     )       , []     )

            _UpperCamelCase				    =       0Xe0_06
            _UpperCamelCase				    =       chr(A_     )
            setattr(A_       , '''additional_special_tokens_ids'''       , [additional_special_token_id]     )
            self.assertListEqual(getattr(A_       , '''additional_special_tokens'''     )       , [additional_special_token]     )
            self.assertListEqual(getattr(A_       , '''additional_special_tokens_ids'''     )       , [additional_special_token_id]     )





       def snake_case__						(							self   :		str     )    ->     Optional[int]:

            '''simple docstring'''




            pass





       def snake_case__						(							self   :		str     )    ->     Optional[Any]:

            '''simple docstring'''




            pass





       def snake_case__						(							self   :		Optional[int]     )    ->     Optional[int]:

            '''simple docstring'''




            pass





       def snake_case__						(							self   :		Any     )    ->     str:

            '''simple docstring'''




            pass





       def snake_case__						(							self   :		Any     )    ->     List[Any]:

            '''simple docstring'''




            pass





       def snake_case__						(							self   :		Optional[int]     )    ->     List[Any]:

            '''simple docstring'''




            pass





       def snake_case__						(							self   :		str     )    ->     Union[str, Any]:

            '''simple docstring'''




            pass







       def snake_case__						(							self   :		List[str]     )    ->     List[Any]:

            '''simple docstring'''




            pass










import inspect
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
__lowerCamelCase       :     str   =     """src/transformers"""


# This is to make sure the transformers module imported is the one in the repo.
__lowerCamelCase       :     Tuple   =     direct_transformers_import(PATH_TO_TRANSFORMERS)

__lowerCamelCase       :     List[str]   =     transformers.models.auto.configuration_auto.CONFIG_MAPPING

# Regex pattern used to find the checkpoint mentioned in the docstring of `config_class`.
# For example, `[bert-base-uncased](https://huggingface.co/bert-base-uncased)`
__lowerCamelCase       :     Optional[Any]   =     re.compile(r"""\[(.+?)\]\((https://huggingface\.co/.+?)\)""")


__lowerCamelCase       :     List[str]   =     {
    """DecisionTransformerConfig""",
    """EncoderDecoderConfig""",
    """MusicgenConfig""",
    """RagConfig""",
    """SpeechEncoderDecoderConfig""",
    """TimmBackboneConfig""",
    """VisionEncoderDecoderConfig""",
    """VisionTextDualEncoderConfig""",
    """LlamaConfig""",
}



def 		A_				(     _lowerCAmelCase     )						-> List[str]:
 UpperCamelCase	:					Optional[Any]       	=   None

 # source code of `config_class`
 UpperCamelCase	:					Tuple       	=   inspect.getsource(_lowerCAmelCase     )
 UpperCamelCase	:					Optional[Any]       	=   _re_checkpoint.findall(_lowerCAmelCase     )

 # Each `checkpoint` is a tuple of a checkpoint name and a checkpoint link.
 # For example, `('bert-base-uncased', 'https://huggingface.co/bert-base-uncased')`
 for ckpt_name, ckpt_link in checkpoints:
  # allow the link to end with `/`
  if ckpt_link.endswith("/"     ):
   UpperCamelCase	:					Dict       	=   ckpt_link[:-1]

  # verify the checkpoint name corresponds to the checkpoint link
  UpperCamelCase	:					Any       	=   F"""https://huggingface.co/{ckpt_name}"""
  if ckpt_link == ckpt_link_from_name:
   UpperCamelCase	:					List[Any]       	=   ckpt_name
   break

 return checkpoint



def 		A_				(     )						-> List[str]:
 UpperCamelCase	:					Optional[int]       	=   []

 for config_class in list(CONFIG_MAPPING.values()     ):
  # Skip deprecated models
  if "models.deprecated" in config_class.__module__:
   continue
  UpperCamelCase	:					Union[str, Any]       	=   get_checkpoint_from_config_class(_lowerCAmelCase     )

  UpperCamelCase	:					Optional[int]       	=   config_class.__name__
  if checkpoint is None and name not in CONFIG_CLASSES_TO_IGNORE_FOR_DOCSTRING_CHECKPOINT_CHECK:
   configs_without_checkpoint.append(_lowerCAmelCase     )

 if len(_lowerCAmelCase     ) > 0:
  UpperCamelCase	:					Any       	=   "\n".join(sorted(_lowerCAmelCase     )     )
  raise ValueError(F"""The following configurations don't contain any valid checkpoint:\n{message}"""     )


if __name__ == "__main__":
    check_config_docstrings_have_checkpoints()
