



from __future__ import annotations

import inspect
import unittest
from typing import List, Tuple

from transformers import ConvNextConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers import TFConvNextForImageClassification, TFConvNextModel


if is_vision_available():
     from PIL import Image

     from transformers import ConvNextImageProcessor



class 		_lowercase		:

   def __init__(   self      :							int				, __lowerCAmelCase      :							str				, __lowerCAmelCase      :							Optional[int]=13				, __lowerCAmelCase      :							str=32				, __lowerCAmelCase      :							Any=3				, __lowerCAmelCase      :							str=4				, __lowerCAmelCase      :							List[str]=[10, 20, 30, 40]				, __lowerCAmelCase      :							List[str]=[2, 2, 3, 2]				, __lowerCAmelCase      :							Any=True				, __lowerCAmelCase      :							int=True				, __lowerCAmelCase      :							Tuple=37				, __lowerCAmelCase      :							Dict="gelu"				, __lowerCAmelCase      :							str=10				, __lowerCAmelCase      :							Optional[Any]=0.0_2				, __lowerCAmelCase      :							Dict=3				, __lowerCAmelCase      :							str=None				, )							->					str:


    """simple docstring"""






    a							=		parent
    a							=		batch_size
    a							=		image_size
    a							=		num_channels
    a							=		num_stages
    a							=		hidden_sizes
    a							=		depths
    a							=		is_training
    a							=		use_labels
    a							=		intermediate_size
    a							=		hidden_act
    a							=		type_sequence_label_size
    a							=		initializer_range
    a							=		scope

   def 		A				(   self      :							Union[str, Any]				)							->					Any:


    """simple docstring"""






    a							=		floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]				)

    a							=		None
    if self.use_labels:
     a							=		ids_tensor([self.batch_size]				, self.type_sequence_label_size				)

    a							=		self.get_config()

    return config, pixel_values, labels

   def 		A				(   self      :							int				)							->					str:


    """simple docstring"""






    return ConvNextConfig(
        num_channels=self.num_channels				, hidden_sizes=self.hidden_sizes				, depths=self.depths				, num_stages=self.num_stages				, hidden_act=self.hidden_act				, is_decoder=__a				, initializer_range=self.initializer_range				, )

   def 		A				(   self      :							int				, __lowerCAmelCase      :							Optional[Any]				, __lowerCAmelCase      :							Optional[int]				, __lowerCAmelCase      :							Tuple				)							->					Any:


    """simple docstring"""






    a							=		TFConvNextModel(config=__a				)
    a							=		model(__a				, training=__a				)
    # expected last hidden states: B, C, H // 32, W // 32
    self.parent.assertEqual(
        result.last_hidden_state.shape				, (self.batch_size, self.hidden_sizes[-1], self.image_size // 32, self.image_size // 32)				, )

   def 		A				(   self      :							Union[str, Any]				, __lowerCAmelCase      :							Optional[Any]				, __lowerCAmelCase      :							List[str]				, __lowerCAmelCase      :							Union[str, Any]				)							->					Tuple:


    """simple docstring"""






    a							=		self.type_sequence_label_size
    a							=		TFConvNextForImageClassification(__a				)
    a							=		model(__a				, labels=__a				, training=__a				)
    self.parent.assertEqual(result.logits.shape				, (self.batch_size, self.type_sequence_label_size)				)




   def 		A				(   self      :							Dict				)							->					List[str]:


    """simple docstring"""






    a							=		self.prepare_config_and_inputs()
    a							=		config_and_inputs
    a							=		{'pixel_values': pixel_values}
    return config, inputs_dict



@require_tf
class 		_lowercase		(							UpperCamelCase_, UpperCamelCase_, unittest.TestCase						):
   _UpperCAmelCase  						=      (TFConvNextModel, TFConvNextForImageClassification) if is_tf_available() else ()
   _UpperCAmelCase  						=      (
       {'''feature-extraction''': TFConvNextModel, '''image-classification''': TFConvNextForImageClassification}
       if is_tf_available()
       else {}
   )

   _UpperCAmelCase  						=      False
   _UpperCAmelCase  						=      False
   _UpperCAmelCase  						=      False
   _UpperCAmelCase  						=      False
   _UpperCAmelCase  						=      False

   def 		A				(   self      :							str				)							->					List[Any]:


    """simple docstring"""






    a							=		TFConvNextModelTester(self				)
    a							=		ConfigTester(
        self				, config_class=__a				, has_text_modality=__a				, hidden_size=37				, )

   @unittest.skip(reason="ConvNext does not use inputs_embeds"				)
   def 		A				(   self      :							List[str]				)							->					List[Any]:


    """simple docstring"""






    pass

   @unittest.skipIf(
       not is_tf_available() or len(tf.config.list_physical_devices("GPU"				)				) == 0				, reason="TF does not support backprop for grouped convolutions on CPU."				, )
   @slow
   def 		A				(   self      :							Any				)							->					Union[str, Any]:


    """simple docstring"""






    super().test_keras_fit()

   @unittest.skip(reason="ConvNext does not support input and output embeddings"				)
   def 		A				(   self      :							Union[str, Any]				)							->					Optional[int]:


    """simple docstring"""






    pass

   def 		A				(   self      :							Dict				)							->					Dict:


    """simple docstring"""






    a							=		self.model_tester.prepare_config_and_inputs_for_common()

    for model_class in self.all_model_classes:
     a							=		model_class(__a				)
     a							=		inspect.signature(model.call				)
     # signature.parameters is an OrderedDict => so arg_names order is deterministic
     a							=		[*signature.parameters.keys()]

     a							=		['pixel_values']
     self.assertListEqual(arg_names[:1]				, __a				)

   def 		A				(   self      :							int				)							->					List[str]:


    """simple docstring"""






    a							=		self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_model(*__a				)

   @unittest.skipIf(
       not is_tf_available() or len(tf.config.list_physical_devices("GPU"				)				) == 0				, reason="TF does not support backprop for grouped convolutions on CPU."				, )
   def 		A				(   self      :							Optional[int]				)							->					int:


    """simple docstring"""






    super().test_dataset_conversion()

   def 		A				(   self      :							List[str]				)							->					int:


    """simple docstring"""






    def check_hidden_states_output(__lowerCAmelCase      :							Tuple				, __lowerCAmelCase      :							List[Any]				, __lowerCAmelCase      :							Dict				):
     a							=		model_class(__a				)

     a							=		model(**self._prepare_for_class(__a				, __a				)				)
     a							=		outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

     a							=		self.model_tester.num_stages
     self.assertEqual(len(__a				)				, expected_num_stages + 1				)

     # ConvNext's feature maps are of shape (batch_size, num_channels, height, width)
     self.assertListEqual(
         list(hidden_states[0].shape[-2:]				)				, [self.model_tester.image_size // 4, self.model_tester.image_size // 4]				, )

    a							=		self.model_tester.prepare_config_and_inputs_for_common()

    for model_class in self.all_model_classes:
     a							=		True
     check_hidden_states_output(__a				, __a				, __a				)

     # check that output_hidden_states also work using config
     del inputs_dict["output_hidden_states"]
     a							=		True

     check_hidden_states_output(__a				, __a				, __a				)

   def 		A				(   self      :							int				)							->					Any:


    """simple docstring"""






    a							=		self.model_tester.prepare_config_and_inputs_for_common()

    def check_equivalence(__lowerCAmelCase      :							Optional[int]				, __lowerCAmelCase      :							Dict				, __lowerCAmelCase      :							str				, __lowerCAmelCase      :							Tuple={}				):
     a							=		model(__a				, return_dict=__a				, **__a				)
     a							=		model(__a				, return_dict=__a				, **__a				).to_tuple()

     def recursive_check(__lowerCAmelCase      :							Union[str, Any]				, __lowerCAmelCase      :							str				):
      if isinstance(__a				, (List, Tuple)				):
       for tuple_iterable_value, dict_iterable_value in zip(__a				, __a				):
        recursive_check(__a				, __a				)
      elif tuple_object is None:
       return
      else:
       self.assertTrue(
           all(tf.equal(__a				, __a				)				)				, msg=(
               "Tuple and dict output are not equal. Difference:"
               f""" {tf.math.reduce_max(tf.abs(tuple_object - dict_object				)				)}"""
           )				, )

      recursive_check(__a				, __a				)

    for model_class in self.all_model_classes:
     a							=		model_class(__a				)

     a							=		self._prepare_for_class(__a				, __a				)
     a							=		self._prepare_for_class(__a				, __a				)
     check_equivalence(__a				, __a				, __a				)

     a							=		self._prepare_for_class(__a				, __a				, return_labels=__a				)
     a							=		self._prepare_for_class(__a				, __a				, return_labels=__a				)
     check_equivalence(__a				, __a				, __a				)

     a							=		self._prepare_for_class(__a				, __a				)
     a							=		self._prepare_for_class(__a				, __a				)
     check_equivalence(__a				, __a				, __a				, {"output_hidden_states": True}				)

     a							=		self._prepare_for_class(__a				, __a				, return_labels=__a				)
     a							=		self._prepare_for_class(__a				, __a				, return_labels=__a				)
     check_equivalence(__a				, __a				, __a				, {"output_hidden_states": True}				)

   def 		A				(   self      :							Dict				)							->					Tuple:


    """simple docstring"""






    a							=		self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_image_classification(*__a				)




   @slow
   def 		A				(   self      :							Dict				)							->					Optional[int]:


    """simple docstring"""






    a							=		TFConvNextModel.from_pretrained("facebook/convnext-tiny-224"				)
    self.assertIsNotNone(__a				)


def 						UpperCAmelCase__   (   ):






 '''simple docstring'''

 a							=		Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"       )
 return image




@require_tf
@require_vision
class 		_lowercase		(							unittest.TestCase						):

   @cached_property
   def 		A				(   self      :							Tuple				)							->					Dict:


    """simple docstring"""






    return ConvNextImageProcessor.from_pretrained("facebook/convnext-tiny-224"				) if is_vision_available() else None




   @slow
   def 		A				(   self      :							int				)							->					Optional[int]:


    """simple docstring"""






    a							=		TFConvNextForImageClassification.from_pretrained("facebook/convnext-tiny-224"				)

    a							=		self.default_image_processor
    a							=		prepare_img()
    a							=		image_processor(images=__a				, return_tensors="tf"				)

    # forward pass
    a							=		model(**__a				)

    # verify the logits
    a							=		tf.TensorShape((1, 1000)				)
    self.assertEqual(outputs.logits.shape				, __a				)

    a							=		tf.constant([-0.0_2_6_0, -0.4_7_3_9, 0.1_9_1_1]				)

    tf.debugging.assert_near(outputs.logits[0, :3]				, __a				, atol=1E-4				)









'''simple docstring'''






import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class a_     (     UpperCamelCase_		,   unittest.TestCase				):
   _snake_case	   =				ElectraTokenizer
   _snake_case	   =				ElectraTokenizerFast
   _snake_case	   =				True
   _snake_case	   =				True
   _snake_case	   =				filter_non_english







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Tuple:
      """simple docstring"""






      super().setUp()

      __snake_case					:   List[Any]  				=			[
          '[UNK]',
          '[CLS]',
          '[SEP]',
          '[PAD]',
          '[MASK]',
          'want',
          '##want',
          '##ed',
          'wa',
          'un',
          'runn',
          '##ing',
          ',',
          'low',
          'lowest',
      ]
      __snake_case					:   int  				=			os.path.join(self.tmpdirname       , VOCAB_FILES_NAMES['vocab_file'])
      with open(self.vocab_file       , 'w'       , encoding='utf-8') as vocab_writer:
         vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]))







   def      SCREAMING_SNAKE_CASE__       (self       , __a)      ->   int:
      """simple docstring"""






      __snake_case					:   Union[str, Any]  				=			'UNwant\u00E9d,running'
      __snake_case					:   Tuple  				=			'unwanted, running'
      return input_text, output_text







   def      SCREAMING_SNAKE_CASE__       (self)      ->   List[str]:
      """simple docstring"""






      __snake_case					:   Union[str, Any]  				=			self.tokenizer_class(self.vocab_file)

      __snake_case					:   str  				=			tokenizer.tokenize('UNwant\u00E9d,running')
      self.assertListEqual(__a       , ['un', '##want', '##ed', ',', 'runn', '##ing'])
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__a)       , [9, 6, 7, 1_2, 1_0, 1_1])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   List[str]:
      """simple docstring"""






      if not self.test_rust_tokenizer:
         return

      __snake_case					:   Union[str, Any]  				=			self.get_tokenizer()
      __snake_case					:   Dict  				=			self.get_rust_tokenizer()

      __snake_case					:   Dict  				=			'UNwant\u00E9d,running'

      __snake_case					:   Tuple  				=			tokenizer.tokenize(__a)
      __snake_case					:   List[Any]  				=			rust_tokenizer.tokenize(__a)
      self.assertListEqual(__a       , __a)

      __snake_case					:   Any  				=			tokenizer.encode(__a       , add_special_tokens=__a)
      __snake_case					:   int  				=			rust_tokenizer.encode(__a       , add_special_tokens=__a)
      self.assertListEqual(__a       , __a)

      __snake_case					:   List[str]  				=			self.get_rust_tokenizer()
      __snake_case					:   Optional[int]  				=			tokenizer.encode(__a)
      __snake_case					:   Dict  				=			rust_tokenizer.encode(__a)
      self.assertListEqual(__a       , __a)

      # With lower casing
      __snake_case					:   Union[str, Any]  				=			self.get_tokenizer(do_lower_case=__a)
      __snake_case					:   List[Any]  				=			self.get_rust_tokenizer(do_lower_case=__a)

      __snake_case					:   List[str]  				=			'UNwant\u00E9d,running'

      __snake_case					:   List[Any]  				=			tokenizer.tokenize(__a)
      __snake_case					:   Union[str, Any]  				=			rust_tokenizer.tokenize(__a)
      self.assertListEqual(__a       , __a)

      __snake_case					:   List[str]  				=			tokenizer.encode(__a       , add_special_tokens=__a)
      __snake_case					:   int  				=			rust_tokenizer.encode(__a       , add_special_tokens=__a)
      self.assertListEqual(__a       , __a)

      __snake_case					:   List[Any]  				=			self.get_rust_tokenizer()
      __snake_case					:   Optional[Any]  				=			tokenizer.encode(__a)
      __snake_case					:   Tuple  				=			rust_tokenizer.encode(__a)
      self.assertListEqual(__a       , __a)







   def      SCREAMING_SNAKE_CASE__       (self)      ->   int:
      """simple docstring"""






      __snake_case					:   int  				=			BasicTokenizer()

      self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz')       , ['ah', '\u535A', '\u63A8', 'zz'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Tuple:
      """simple docstring"""






      __snake_case					:   Optional[Any]  				=			BasicTokenizer(do_lower_case=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')       , ['hello', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')       , ['hello'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Tuple:
      """simple docstring"""






      __snake_case					:   Any  				=			BasicTokenizer(do_lower_case=__a       , strip_accents=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')       , ['hällo', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')       , ['h\u00E9llo'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   str:
      """simple docstring"""






      __snake_case					:   Optional[Any]  				=			BasicTokenizer(do_lower_case=__a       , strip_accents=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')       , ['hallo', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')       , ['hello'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   str:
      """simple docstring"""






      __snake_case					:   Tuple  				=			BasicTokenizer(do_lower_case=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')       , ['hallo', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')       , ['hello'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Union[str, Any]:
      """simple docstring"""






      __snake_case					:   Dict  				=			BasicTokenizer(do_lower_case=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')       , ['HeLLo', '!', 'how', 'Are', 'yoU', '?'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   int:
      """simple docstring"""






      __snake_case					:   Union[str, Any]  				=			BasicTokenizer(do_lower_case=__a       , strip_accents=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')       , ['HäLLo', '!', 'how', 'Are', 'yoU', '?'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   int:
      """simple docstring"""






      __snake_case					:   List[Any]  				=			BasicTokenizer(do_lower_case=__a       , strip_accents=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')       , ['HaLLo', '!', 'how', 'Are', 'yoU', '?'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   str:
      """simple docstring"""






      __snake_case					:   Union[str, Any]  				=			BasicTokenizer(do_lower_case=__a       , never_split=['[UNK]'])

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]')       , ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   List[str]:
      """simple docstring"""






      __snake_case					:   Tuple  				=			['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

      __snake_case					:   List[str]  				=			{}
      for i, token in enumerate(__a):
         __snake_case					:   str  				=			i
      __snake_case					:   List[str]  				=			WordpieceTokenizer(vocab=__a       , unk_token='[UNK]')

      self.assertListEqual(tokenizer.tokenize('')       , [])

      self.assertListEqual(tokenizer.tokenize('unwanted running')       , ['un', '##want', '##ed', 'runn', '##ing'])

      self.assertListEqual(tokenizer.tokenize('unwantedX running')       , ['[UNK]', 'runn', '##ing'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Tuple:
      """simple docstring"""






      self.assertTrue(_is_whitespace(' '))
      self.assertTrue(_is_whitespace('\t'))
      self.assertTrue(_is_whitespace('\r'))
      self.assertTrue(_is_whitespace('\n'))
      self.assertTrue(_is_whitespace('\u00A0'))

      self.assertFalse(_is_whitespace('A'))
      self.assertFalse(_is_whitespace('-'))







   def      SCREAMING_SNAKE_CASE__       (self)      ->   List[str]:
      """simple docstring"""






      self.assertTrue(_is_control('\u0005'))

      self.assertFalse(_is_control('A'))
      self.assertFalse(_is_control(' '))
      self.assertFalse(_is_control('\t'))
      self.assertFalse(_is_control('\r'))







   def      SCREAMING_SNAKE_CASE__       (self)      ->   int:
      """simple docstring"""






      self.assertTrue(_is_punctuation('-'))
      self.assertTrue(_is_punctuation('$'))
      self.assertTrue(_is_punctuation('`'))
      self.assertTrue(_is_punctuation('.'))

      self.assertFalse(_is_punctuation('A'))
      self.assertFalse(_is_punctuation(' '))







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Optional[Any]:
      """simple docstring"""






      __snake_case					:   Dict  				=			self.get_tokenizer()
      __snake_case					:   Dict  				=			self.get_rust_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(__a) for t in ['Test', '\xad', 'test']]       , [['[UNK]'], [], ['[UNK]']])

      self.assertListEqual(
          [rust_tokenizer.tokenize(__a) for t in ['Test', '\xad', 'test']]       , [['[UNK]'], [], ['[UNK]']])







   @slow
   def      SCREAMING_SNAKE_CASE__       (self)      ->   Tuple:
      """simple docstring"""






      __snake_case					:   str  				=			self.tokenizer_class.from_pretrained('google/electra-base-discriminator')

      __snake_case					:   Optional[Any]  				=			tokenizer.encode('sequence builders'       , add_special_tokens=__a)
      __snake_case					:   List[Any]  				=			tokenizer.encode('multi-sequence build'       , add_special_tokens=__a)

      __snake_case					:   Tuple  				=			tokenizer.build_inputs_with_special_tokens(__a)
      __snake_case					:   int  				=			tokenizer.build_inputs_with_special_tokens(__a       , __a)

      assert encoded_sentence == [1_0_1] + text + [1_0_2]
      assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Optional[int]:
      """simple docstring"""






      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
            __snake_case					:   str  				=			self.rust_tokenizer_class.from_pretrained(__a       , **__a)

            __snake_case					:   List[Any]  				=			F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
            __snake_case					:   str  				=			tokenizer_r.encode_plus(
                __a       , return_attention_mask=__a       , return_token_type_ids=__a       , return_offsets_mapping=__a       , add_special_tokens=__a       , )

            __snake_case					:   int  				=			tokenizer_r.do_lower_case if hasattr(__a       , 'do_lower_case') else False
            __snake_case					:   Any  				=			(
                [
                    ((0, 0), tokenizer_r.cls_token),
                    ((0, 1), 'A'),
                    ((1, 2), ','),
                    ((3, 5), 'na'),
                    ((5, 6), '##ï'),
                    ((6, 8), '##ve'),
                    ((9, 1_5), tokenizer_r.mask_token),
                    ((1_6, 2_1), 'Allen'),
                    ((2_1, 2_3), '##NL'),
                    ((2_3, 2_4), '##P'),
                    ((2_5, 3_3), 'sentence'),
                    ((3_3, 3_4), '.'),
                    ((0, 0), tokenizer_r.sep_token),
                ]
                if not do_lower_case
                else [
                    ((0, 0), tokenizer_r.cls_token),
                    ((0, 1), 'a'),
                    ((1, 2), ','),
                    ((3, 8), 'naive'),
                    ((9, 1_5), tokenizer_r.mask_token),
                    ((1_6, 2_1), 'allen'),
                    ((2_1, 2_3), '##nl'),
                    ((2_3, 2_4), '##p'),
                    ((2_5, 3_3), 'sentence'),
                    ((3_3, 3_4), '.'),
                    ((0, 0), tokenizer_r.sep_token),
                ]
            )

            self.assertEqual(
                [e[1] for e in expected_results]       , tokenizer_r.convert_ids_to_tokens(tokens['input_ids']))
            self.assertEqual([e[0] for e in expected_results]       , tokens['offset_mapping'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Any:
      """simple docstring"""






      __snake_case					:   str  				=			['的', '人', '有']
      __snake_case					:   str  				=			''.join(__a)
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
            __snake_case					:   Tuple  				=			True
            __snake_case					:   str  				=			self.tokenizer_class.from_pretrained(__a       , **__a)
            __snake_case					:   Optional[int]  				=			self.rust_tokenizer_class.from_pretrained(__a       , **__a)

            __snake_case					:   List[str]  				=			tokenizer_p.encode(__a       , add_special_tokens=__a)
            __snake_case					:   Optional[Any]  				=			tokenizer_r.encode(__a       , add_special_tokens=__a)

            __snake_case					:   Union[str, Any]  				=			tokenizer_r.convert_ids_to_tokens(__a)
            __snake_case					:   List[Any]  				=			tokenizer_p.convert_ids_to_tokens(__a)

            # it is expected that each Chinese character is not preceded by "##"
            self.assertListEqual(__a       , __a)
            self.assertListEqual(__a       , __a)

            __snake_case					:   Optional[int]  				=			False
            __snake_case					:   Any  				=			self.rust_tokenizer_class.from_pretrained(__a       , **__a)
            __snake_case					:   List[str]  				=			self.tokenizer_class.from_pretrained(__a       , **__a)

            __snake_case					:   List[str]  				=			tokenizer_r.encode(__a       , add_special_tokens=__a)
            __snake_case					:   Optional[Any]  				=			tokenizer_p.encode(__a       , add_special_tokens=__a)

            __snake_case					:   List[str]  				=			tokenizer_r.convert_ids_to_tokens(__a)
            __snake_case					:   Optional[int]  				=			tokenizer_p.convert_ids_to_tokens(__a)

            # it is expected that only the first Chinese character is not preceded by "##".
            __snake_case					:   Any  				=			[
                F"""##{token}""" if idx != 0 else token for idx, token in enumerate(__a)
            ]
            self.assertListEqual(__a       , __a)
            self.assertListEqual(__a       , __a)