




'''simple docstring'''


import inspect
import unittest

from transformers import SwinaSRConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch
     from torch import nn

     from transformers import SwinaSRForImageSuperResolution, SwinaSRModel
     from transformers.models.swinasr.modeling_swinasr import SWIN2SR_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
     from PIL import Image

     from transformers import SwinaSRImageProcessor






class 						lowerCAmelCase_	:







  '''simple docstring'''

  def __init__( self				:							Union[str, Any]					,  SCREAMING_SNAKE_CASE_				:							Optional[int]					,  SCREAMING_SNAKE_CASE_				:							Union[str, Any]=13					,  SCREAMING_SNAKE_CASE_				:							Union[str, Any]=32					,  SCREAMING_SNAKE_CASE_				:							List[str]=1					,  SCREAMING_SNAKE_CASE_				:							Optional[Any]=3					,  SCREAMING_SNAKE_CASE_				:							Optional[Any]=16					,  SCREAMING_SNAKE_CASE_				:							int=[1, 2, 1]					,  SCREAMING_SNAKE_CASE_				:							Any=[2, 2, 4]					,  SCREAMING_SNAKE_CASE_				:							List[Any]=2					,  SCREAMING_SNAKE_CASE_				:							int=2.0					,  SCREAMING_SNAKE_CASE_				:							Dict=True					,  SCREAMING_SNAKE_CASE_				:							Optional[Any]=0.0					,  SCREAMING_SNAKE_CASE_				:							List[Any]=0.0					,  SCREAMING_SNAKE_CASE_				:							Union[str, Any]=0.1					,  SCREAMING_SNAKE_CASE_				:							Any="gelu"					,  SCREAMING_SNAKE_CASE_				:							Optional[int]=False					,  SCREAMING_SNAKE_CASE_				:							List[str]=True					,  SCREAMING_SNAKE_CASE_				:							List[Any]=0.02					,  SCREAMING_SNAKE_CASE_				:							str=1E-5					,  SCREAMING_SNAKE_CASE_				:							Dict=True					,  SCREAMING_SNAKE_CASE_				:							Dict=None					,  SCREAMING_SNAKE_CASE_				:							str=False					,  SCREAMING_SNAKE_CASE_				:							Union[str, Any]=2					,  )			->		str:







       '''simple docstring'''



       A:		Any          =		parent
       A:		Tuple          =		batch_size
       A:		Any          =		image_size
       A:		Optional[Any]          =		patch_size
       A:		Optional[int]          =		num_channels
       A:		int          =		embed_dim
       A:		Any          =		depths
       A:		int          =		num_heads
       A:		Optional[int]          =		window_size
       A:		List[Any]          =		mlp_ratio
       A:		Optional[int]          =		qkv_bias
       A:		int          =		hidden_dropout_prob
       A:		Dict          =		attention_probs_dropout_prob
       A:		str          =		drop_path_rate
       A:		Dict          =		hidden_act
       A:		Optional[Any]          =		use_absolute_embeddings
       A:		Tuple          =		patch_norm
       A:		Any          =		layer_norm_eps
       A:		List[str]          =		initializer_range
       A:		Union[str, Any]          =		is_training
       A:		List[Any]          =		scope
       A:		Optional[int]          =		use_labels
       A:		Union[str, Any]          =		upscale

       # here we set some attributes to make tests pass
       A:		Any          =		len(__a	)
       A:		int          =		embed_dim
       A:		List[str]          =		(image_size // patch_size) ** 2



  def 					_snake_case     ( self				:							Tuple	)			->		List[Any]:







       '''simple docstring'''



       A:		List[Any]          =		floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]	)

       A:		str          =		None
       if self.use_labels:
            A:		Optional[Any]          =		ids_tensor([self.batch_size]					,  self.type_sequence_label_size	)

       A:		Tuple          =		self.get_config()

       return config, pixel_values, labels



  def 					_snake_case     ( self				:							Tuple	)			->		List[Any]:







       '''simple docstring'''



       return SwinaSRConfig(
           image_size=self.image_size					,  patch_size=self.patch_size					,  num_channels=self.num_channels					,  embed_dim=self.embed_dim					,  depths=self.depths					,  num_heads=self.num_heads					,  window_size=self.window_size					,  mlp_ratio=self.mlp_ratio					,  qkv_bias=self.qkv_bias					,  hidden_dropout_prob=self.hidden_dropout_prob					,  attention_probs_dropout_prob=self.attention_probs_dropout_prob					,  drop_path_rate=self.drop_path_rate					,  hidden_act=self.hidden_act					,  use_absolute_embeddings=self.use_absolute_embeddings					,  path_norm=self.patch_norm					,  layer_norm_eps=self.layer_norm_eps					,  initializer_range=self.initializer_range					,  upscale=self.upscale					,  )



  def 					_snake_case     ( self				:							str					,  SCREAMING_SNAKE_CASE_				:							Any					,  SCREAMING_SNAKE_CASE_				:							Dict					,  SCREAMING_SNAKE_CASE_				:							List[str]	)			->		Tuple:







       '''simple docstring'''



       A:		Optional[Any]          =		SwinaSRModel(config=__a	)
       model.to(__a	)
       model.eval()
       A:		Any          =		model(__a	)

       self.parent.assertEqual(
           result.last_hidden_state.shape					,  (self.batch_size, self.embed_dim, self.image_size, self.image_size)	)



  def 					_snake_case     ( self				:							Optional[Any]					,  SCREAMING_SNAKE_CASE_				:							Any					,  SCREAMING_SNAKE_CASE_				:							str					,  SCREAMING_SNAKE_CASE_				:							Tuple	)			->		Optional[int]:







       '''simple docstring'''



       A:		Union[str, Any]          =		SwinaSRForImageSuperResolution(__a	)
       model.to(__a	)
       model.eval()
       A:		Optional[Any]          =		model(__a	)

       A:		int          =		self.image_size * self.upscale
       self.parent.assertEqual(
           result.reconstruction.shape					,  (self.batch_size, self.num_channels, expected_image_size, expected_image_size)	)




  def 					_snake_case     ( self				:							Optional[Any]	)			->		Dict:







       '''simple docstring'''



       A:		Optional[int]          =		self.prepare_config_and_inputs()
       A:		Optional[int]          =		config_and_inputs
       A:		List[str]          =		{'pixel_values': pixel_values}
       return config, inputs_dict






@require_torch
class 						lowerCAmelCase_	(       UpperCamelCase_       ,					UpperCamelCase_       ,					unittest.TestCase							):







  '''simple docstring'''

  UpperCamelCase_			:      Tuple										=				(SwinaSRModel, SwinaSRForImageSuperResolution) if is_torch_available() else ()
  UpperCamelCase_			:      str										=				{"""feature-extraction""": SwinaSRModel} if is_torch_available() else {}

  UpperCamelCase_			:      Any										=				False
  UpperCamelCase_			:      str										=				False
  UpperCamelCase_			:      List[str]										=				False
  UpperCamelCase_			:      List[str]										=				False
  UpperCamelCase_			:      List[Any]										=				False



  def 					_snake_case     ( self				:							Optional[int]	)			->		str:







       '''simple docstring'''



       A:		Tuple          =		SwinaSRModelTester(self	)
       A:		int          =		ConfigTester(self					,  config_class=__a					,  embed_dim=37	)



  def 					_snake_case     ( self				:							Union[str, Any]	)			->		List[str]:







       '''simple docstring'''



       self.config_tester.create_and_test_config_to_json_string()
       self.config_tester.create_and_test_config_to_json_file()
       self.config_tester.create_and_test_config_from_and_save_pretrained()
       self.config_tester.create_and_test_config_with_num_labels()
       self.config_tester.check_config_can_be_init_without_params()
       self.config_tester.check_config_arguments_init()



  def 					_snake_case     ( self				:							Tuple	)			->		Any:







       '''simple docstring'''



       A:		Tuple          =		self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*__a	)



  def 					_snake_case     ( self				:							Union[str, Any]	)			->		int:







       '''simple docstring'''



       A:		List[str]          =		self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_image_super_resolution(*__a	)



  @unittest.skip(reason='''Got `CUDA error: misaligned address` with PyTorch 2.0.0.'''	)
  def 					_snake_case     ( self				:							Union[str, Any]	)			->		Dict:







       '''simple docstring'''



       pass



  @unittest.skip(reason='''Swin2SR does not use inputs_embeds'''	)
  def 					_snake_case     ( self				:							List[Any]	)			->		Union[str, Any]:







       '''simple docstring'''



       pass



  @unittest.skip(reason='''Swin2SR does not support training yet'''	)
  def 					_snake_case     ( self				:							Optional[Any]	)			->		Dict:







       '''simple docstring'''



       pass



  @unittest.skip(reason='''Swin2SR does not support training yet'''	)
  def 					_snake_case     ( self				:							Tuple	)			->		List[Any]:







       '''simple docstring'''



       pass



  def 					_snake_case     ( self				:							Optional[int]	)			->		Optional[Any]:







       '''simple docstring'''



       A:		Dict          =		self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes:
            A:		Optional[int]          =		model_class(__a	)
            self.assertIsInstance(model.get_input_embeddings()					,  (nn.Module)	)
            A:		Union[str, Any]          =		model.get_output_embeddings()
            self.assertTrue(x is None or isinstance(__a					,  nn.Linear	)	)



  def 					_snake_case     ( self				:							Optional[int]	)			->		int:







       '''simple docstring'''



       A:		Optional[Any]          =		self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes:
            A:		Dict          =		model_class(__a	)
            A:		Dict          =		inspect.signature(model.forward	)
            # signature.parameters is an OrderedDict => so arg_names order is deterministic
            A:		Dict          =		[*signature.parameters.keys()]

            A:		Any          =		['pixel_values']
            self.assertListEqual(arg_names[:1]					,  __a	)



  @slow
  def 					_snake_case     ( self				:							Dict	)			->		Optional[int]:







       '''simple docstring'''



       for model_name in SWIN2SR_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            A:		Optional[Any]          =		SwinaSRModel.from_pretrained(__a	)
            self.assertIsNotNone(__a	)



  def 					_snake_case     ( self				:							Any	)			->		Tuple:







       '''simple docstring'''



       A:		Dict          =		self.model_tester.prepare_config_and_inputs_for_common()

       A:		Union[str, Any]          =		_config_zero_init(__a	)
       for model_class in self.all_model_classes:
            A:		Any          =		model_class(config=__a	)
            for name, param in model.named_parameters():
                 if "logit_scale" in name:
                      continue
                 if param.requires_grad:
                      self.assertIn(
                          ((param.data.mean() * 1E9).round() / 1E9).item()					,  [0.0, 1.0]					,  msg=f"""Parameter {name} of model {model_class} seems not properly initialized"""					,  )




  def 					_snake_case     ( self				:							Tuple	)			->		Tuple:







       '''simple docstring'''



       A:		int          =		self.model_tester.prepare_config_and_inputs_for_common()
       A:		str          =		True

       for model_class in self.all_model_classes:
            A:		Any          =		True
            A:		Tuple          =		False
            A:		List[Any]          =		True
            A:		List[Any]          =		model_class(__a	)
            model.to(__a	)
            model.eval()
            with torch.no_grad():
                 A:		str          =		model(**self._prepare_for_class(__a					,  __a	)	)
            A:		List[Any]          =		outputs.attentions
            A:		str          =		len(self.model_tester.depths	)
            self.assertEqual(len(__a	)					,  __a	)

            # check that output_attentions also work using config
            del inputs_dict["output_attentions"]
            A:		str          =		True
            A:		Tuple          =		config.window_size**2
            A:		Optional[Any]          =		model_class(__a	)
            model.to(__a	)
            model.eval()
            with torch.no_grad():
                 A:		str          =		model(**self._prepare_for_class(__a					,  __a	)	)
            A:		List[str]          =		outputs.attentions
            self.assertEqual(len(__a	)					,  __a	)

            self.assertListEqual(
                list(attentions[0].shape[-3:]	)					,  [self.model_tester.num_heads[0], window_size_squared, window_size_squared]					,  )
            A:		Union[str, Any]          =		len(__a	)

            # Check attention is always last and order is fine
            A:		Dict          =		True
            A:		Tuple          =		True
            A:		Optional[Any]          =		model_class(__a	)
            model.to(__a	)
            model.eval()
            with torch.no_grad():
                 A:		Any          =		model(**self._prepare_for_class(__a					,  __a	)	)

            self.assertEqual(out_len + 1					,  len(__a	)	)

            A:		Dict          =		outputs.attentions

            self.assertEqual(len(__a	)					,  __a	)

            self.assertListEqual(
                list(self_attentions[0].shape[-3:]	)					,  [self.model_tester.num_heads[0], window_size_squared, window_size_squared]					,  )






@require_vision
@require_torch
@slow
class 						lowerCAmelCase_	(       unittest.TestCase							):







  '''simple docstring'''

  def 					_snake_case     ( self				:							List[Any]	)			->		List[str]:







       '''simple docstring'''



       A:		Optional[int]          =		SwinaSRImageProcessor()
       A:		int          =		SwinaSRForImageSuperResolution.from_pretrained('''caidas/swin2SR-classical-sr-x2-64'''	).to(__a	)

       A:		Optional[Any]          =		Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''	)
       A:		List[str]          =		processor(images=__a					,  return_tensors='''pt'''	).to(__a	)

       # forward pass
       with torch.no_grad():
            A:		Union[str, Any]          =		model(**__a	)

       # verify the logits
       A:		Dict          =		torch.Size([1, 3, 9_76, 12_96]	)
       self.assertEqual(outputs.reconstruction.shape					,  __a	)
       A:		int          =		torch.tensor(
           [[0.5458, 0.5546, 0.5638], [0.5526, 0.5565, 0.5651], [0.5396, 0.5426, 0.5621]]	).to(__a	)
       self.assertTrue(torch.allclose(outputs.reconstruction[0, 0, :3, :3]					,  __a					,  atol=1E-4	)	)














'''simple docstring'''






import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class a_     (     UpperCamelCase_		,   unittest.TestCase				):
   _snake_case	   =				ElectraTokenizer
   _snake_case	   =				ElectraTokenizerFast
   _snake_case	   =				True
   _snake_case	   =				True
   _snake_case	   =				filter_non_english







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Tuple:
      """simple docstring"""






      super().setUp()

      __snake_case					:   List[Any]  				=			[
          '[UNK]',
          '[CLS]',
          '[SEP]',
          '[PAD]',
          '[MASK]',
          'want',
          '##want',
          '##ed',
          'wa',
          'un',
          'runn',
          '##ing',
          ',',
          'low',
          'lowest',
      ]
      __snake_case					:   int  				=			os.path.join(self.tmpdirname       , VOCAB_FILES_NAMES['vocab_file'])
      with open(self.vocab_file       , 'w'       , encoding='utf-8') as vocab_writer:
         vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]))







   def      SCREAMING_SNAKE_CASE__       (self       , __a)      ->   int:
      """simple docstring"""






      __snake_case					:   Union[str, Any]  				=			'UNwant\u00E9d,running'
      __snake_case					:   Tuple  				=			'unwanted, running'
      return input_text, output_text







   def      SCREAMING_SNAKE_CASE__       (self)      ->   List[str]:
      """simple docstring"""






      __snake_case					:   Union[str, Any]  				=			self.tokenizer_class(self.vocab_file)

      __snake_case					:   str  				=			tokenizer.tokenize('UNwant\u00E9d,running')
      self.assertListEqual(__a       , ['un', '##want', '##ed', ',', 'runn', '##ing'])
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__a)       , [9, 6, 7, 1_2, 1_0, 1_1])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   List[str]:
      """simple docstring"""






      if not self.test_rust_tokenizer:
         return

      __snake_case					:   Union[str, Any]  				=			self.get_tokenizer()
      __snake_case					:   Dict  				=			self.get_rust_tokenizer()

      __snake_case					:   Dict  				=			'UNwant\u00E9d,running'

      __snake_case					:   Tuple  				=			tokenizer.tokenize(__a)
      __snake_case					:   List[Any]  				=			rust_tokenizer.tokenize(__a)
      self.assertListEqual(__a       , __a)

      __snake_case					:   Any  				=			tokenizer.encode(__a       , add_special_tokens=__a)
      __snake_case					:   int  				=			rust_tokenizer.encode(__a       , add_special_tokens=__a)
      self.assertListEqual(__a       , __a)

      __snake_case					:   List[str]  				=			self.get_rust_tokenizer()
      __snake_case					:   Optional[int]  				=			tokenizer.encode(__a)
      __snake_case					:   Dict  				=			rust_tokenizer.encode(__a)
      self.assertListEqual(__a       , __a)

      # With lower casing
      __snake_case					:   Union[str, Any]  				=			self.get_tokenizer(do_lower_case=__a)
      __snake_case					:   List[Any]  				=			self.get_rust_tokenizer(do_lower_case=__a)

      __snake_case					:   List[str]  				=			'UNwant\u00E9d,running'

      __snake_case					:   List[Any]  				=			tokenizer.tokenize(__a)
      __snake_case					:   Union[str, Any]  				=			rust_tokenizer.tokenize(__a)
      self.assertListEqual(__a       , __a)

      __snake_case					:   List[str]  				=			tokenizer.encode(__a       , add_special_tokens=__a)
      __snake_case					:   int  				=			rust_tokenizer.encode(__a       , add_special_tokens=__a)
      self.assertListEqual(__a       , __a)

      __snake_case					:   List[Any]  				=			self.get_rust_tokenizer()
      __snake_case					:   Optional[Any]  				=			tokenizer.encode(__a)
      __snake_case					:   Tuple  				=			rust_tokenizer.encode(__a)
      self.assertListEqual(__a       , __a)







   def      SCREAMING_SNAKE_CASE__       (self)      ->   int:
      """simple docstring"""






      __snake_case					:   int  				=			BasicTokenizer()

      self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz')       , ['ah', '\u535A', '\u63A8', 'zz'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Tuple:
      """simple docstring"""






      __snake_case					:   Optional[Any]  				=			BasicTokenizer(do_lower_case=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')       , ['hello', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')       , ['hello'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Tuple:
      """simple docstring"""






      __snake_case					:   Any  				=			BasicTokenizer(do_lower_case=__a       , strip_accents=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')       , ['hällo', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')       , ['h\u00E9llo'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   str:
      """simple docstring"""






      __snake_case					:   Optional[Any]  				=			BasicTokenizer(do_lower_case=__a       , strip_accents=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')       , ['hallo', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')       , ['hello'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   str:
      """simple docstring"""






      __snake_case					:   Tuple  				=			BasicTokenizer(do_lower_case=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')       , ['hallo', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')       , ['hello'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Union[str, Any]:
      """simple docstring"""






      __snake_case					:   Dict  				=			BasicTokenizer(do_lower_case=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')       , ['HeLLo', '!', 'how', 'Are', 'yoU', '?'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   int:
      """simple docstring"""






      __snake_case					:   Union[str, Any]  				=			BasicTokenizer(do_lower_case=__a       , strip_accents=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')       , ['HäLLo', '!', 'how', 'Are', 'yoU', '?'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   int:
      """simple docstring"""






      __snake_case					:   List[Any]  				=			BasicTokenizer(do_lower_case=__a       , strip_accents=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')       , ['HaLLo', '!', 'how', 'Are', 'yoU', '?'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   str:
      """simple docstring"""






      __snake_case					:   Union[str, Any]  				=			BasicTokenizer(do_lower_case=__a       , never_split=['[UNK]'])

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]')       , ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   List[str]:
      """simple docstring"""






      __snake_case					:   Tuple  				=			['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

      __snake_case					:   List[str]  				=			{}
      for i, token in enumerate(__a):
         __snake_case					:   str  				=			i
      __snake_case					:   List[str]  				=			WordpieceTokenizer(vocab=__a       , unk_token='[UNK]')

      self.assertListEqual(tokenizer.tokenize('')       , [])

      self.assertListEqual(tokenizer.tokenize('unwanted running')       , ['un', '##want', '##ed', 'runn', '##ing'])

      self.assertListEqual(tokenizer.tokenize('unwantedX running')       , ['[UNK]', 'runn', '##ing'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Tuple:
      """simple docstring"""






      self.assertTrue(_is_whitespace(' '))
      self.assertTrue(_is_whitespace('\t'))
      self.assertTrue(_is_whitespace('\r'))
      self.assertTrue(_is_whitespace('\n'))
      self.assertTrue(_is_whitespace('\u00A0'))

      self.assertFalse(_is_whitespace('A'))
      self.assertFalse(_is_whitespace('-'))







   def      SCREAMING_SNAKE_CASE__       (self)      ->   List[str]:
      """simple docstring"""






      self.assertTrue(_is_control('\u0005'))

      self.assertFalse(_is_control('A'))
      self.assertFalse(_is_control(' '))
      self.assertFalse(_is_control('\t'))
      self.assertFalse(_is_control('\r'))







   def      SCREAMING_SNAKE_CASE__       (self)      ->   int:
      """simple docstring"""






      self.assertTrue(_is_punctuation('-'))
      self.assertTrue(_is_punctuation('$'))
      self.assertTrue(_is_punctuation('`'))
      self.assertTrue(_is_punctuation('.'))

      self.assertFalse(_is_punctuation('A'))
      self.assertFalse(_is_punctuation(' '))







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Optional[Any]:
      """simple docstring"""






      __snake_case					:   Dict  				=			self.get_tokenizer()
      __snake_case					:   Dict  				=			self.get_rust_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(__a) for t in ['Test', '\xad', 'test']]       , [['[UNK]'], [], ['[UNK]']])

      self.assertListEqual(
          [rust_tokenizer.tokenize(__a) for t in ['Test', '\xad', 'test']]       , [['[UNK]'], [], ['[UNK]']])







   @slow
   def      SCREAMING_SNAKE_CASE__       (self)      ->   Tuple:
      """simple docstring"""






      __snake_case					:   str  				=			self.tokenizer_class.from_pretrained('google/electra-base-discriminator')

      __snake_case					:   Optional[Any]  				=			tokenizer.encode('sequence builders'       , add_special_tokens=__a)
      __snake_case					:   List[Any]  				=			tokenizer.encode('multi-sequence build'       , add_special_tokens=__a)

      __snake_case					:   Tuple  				=			tokenizer.build_inputs_with_special_tokens(__a)
      __snake_case					:   int  				=			tokenizer.build_inputs_with_special_tokens(__a       , __a)

      assert encoded_sentence == [1_0_1] + text + [1_0_2]
      assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Optional[int]:
      """simple docstring"""






      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
            __snake_case					:   str  				=			self.rust_tokenizer_class.from_pretrained(__a       , **__a)

            __snake_case					:   List[Any]  				=			F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
            __snake_case					:   str  				=			tokenizer_r.encode_plus(
                __a       , return_attention_mask=__a       , return_token_type_ids=__a       , return_offsets_mapping=__a       , add_special_tokens=__a       , )

            __snake_case					:   int  				=			tokenizer_r.do_lower_case if hasattr(__a       , 'do_lower_case') else False
            __snake_case					:   Any  				=			(
                [
                    ((0, 0), tokenizer_r.cls_token),
                    ((0, 1), 'A'),
                    ((1, 2), ','),
                    ((3, 5), 'na'),
                    ((5, 6), '##ï'),
                    ((6, 8), '##ve'),
                    ((9, 1_5), tokenizer_r.mask_token),
                    ((1_6, 2_1), 'Allen'),
                    ((2_1, 2_3), '##NL'),
                    ((2_3, 2_4), '##P'),
                    ((2_5, 3_3), 'sentence'),
                    ((3_3, 3_4), '.'),
                    ((0, 0), tokenizer_r.sep_token),
                ]
                if not do_lower_case
                else [
                    ((0, 0), tokenizer_r.cls_token),
                    ((0, 1), 'a'),
                    ((1, 2), ','),
                    ((3, 8), 'naive'),
                    ((9, 1_5), tokenizer_r.mask_token),
                    ((1_6, 2_1), 'allen'),
                    ((2_1, 2_3), '##nl'),
                    ((2_3, 2_4), '##p'),
                    ((2_5, 3_3), 'sentence'),
                    ((3_3, 3_4), '.'),
                    ((0, 0), tokenizer_r.sep_token),
                ]
            )

            self.assertEqual(
                [e[1] for e in expected_results]       , tokenizer_r.convert_ids_to_tokens(tokens['input_ids']))
            self.assertEqual([e[0] for e in expected_results]       , tokens['offset_mapping'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Any:
      """simple docstring"""






      __snake_case					:   str  				=			['的', '人', '有']
      __snake_case					:   str  				=			''.join(__a)
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
            __snake_case					:   Tuple  				=			True
            __snake_case					:   str  				=			self.tokenizer_class.from_pretrained(__a       , **__a)
            __snake_case					:   Optional[int]  				=			self.rust_tokenizer_class.from_pretrained(__a       , **__a)

            __snake_case					:   List[str]  				=			tokenizer_p.encode(__a       , add_special_tokens=__a)
            __snake_case					:   Optional[Any]  				=			tokenizer_r.encode(__a       , add_special_tokens=__a)

            __snake_case					:   Union[str, Any]  				=			tokenizer_r.convert_ids_to_tokens(__a)
            __snake_case					:   List[Any]  				=			tokenizer_p.convert_ids_to_tokens(__a)

            # it is expected that each Chinese character is not preceded by "##"
            self.assertListEqual(__a       , __a)
            self.assertListEqual(__a       , __a)

            __snake_case					:   Optional[int]  				=			False
            __snake_case					:   Any  				=			self.rust_tokenizer_class.from_pretrained(__a       , **__a)
            __snake_case					:   List[str]  				=			self.tokenizer_class.from_pretrained(__a       , **__a)

            __snake_case					:   List[str]  				=			tokenizer_r.encode(__a       , add_special_tokens=__a)
            __snake_case					:   Optional[Any]  				=			tokenizer_p.encode(__a       , add_special_tokens=__a)

            __snake_case					:   List[str]  				=			tokenizer_r.convert_ids_to_tokens(__a)
            __snake_case					:   Optional[int]  				=			tokenizer_p.convert_ids_to_tokens(__a)

            # it is expected that only the first Chinese character is not preceded by "##".
            __snake_case					:   Any  				=			[
                F"""##{token}""" if idx != 0 else token for idx, token in enumerate(__a)
            ]
            self.assertListEqual(__a       , __a)
            self.assertListEqual(__a       , __a)