






'''simple docstring'''






from dataclasses import dataclass
from typing import Optional, Tuple, Union

import torch
import torch.nn as nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .embeddings import GaussianFourierProjection, TimestepEmbedding, Timesteps
from .modeling_utils import ModelMixin
from .unet_ad_blocks import UNetMidBlockaD, get_down_block, get_up_block




@dataclass
class a_     (     UpperCamelCase_				):
   _snake_case	   =				42




class a_     (     UpperCamelCase_		,   UpperCamelCase_				):







   @register_to_config
   def __init__(self       , __a = None       , __a = 3       , __a = 3       , __a = False       , __a = "positional"       , __a = 0       , __a = True       , __a = ("DownBlock2D", "AttnDownBlock2D", "AttnDownBlock2D", "AttnDownBlock2D")       , __a = ("AttnUpBlock2D", "AttnUpBlock2D", "AttnUpBlock2D", "UpBlock2D")       , __a = (2_2_4, 4_4_8, 6_7_2, 8_9_6)       , __a = 2       , __a = 1       , __a = 1       , __a = "conv"       , __a = "conv"       , __a = "silu"       , __a = 8       , __a = 3_2       , __a = 1E-5       , __a = "default"       , __a = True       , __a = None       , __a = None       , )      ->   List[str]:
      """simple docstring"""






      super().__init__()

      __snake_case					:   Optional[int]  				=			sample_size
      __snake_case					:   Union[str, Any]  				=			block_out_channels[0] * 4

      # Check inputs
      if len(__a) != len(__a):
         raise ValueError(
             F"""Must provide the same number of `down_block_types` as `up_block_types`. `down_block_types`: {down_block_types}. `up_block_types`: {up_block_types}.""")

      if len(__a) != len(__a):
         raise ValueError(
             F"""Must provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}.""")

      # input
      __snake_case					:   Union[str, Any]  				=			nn.Convad(__a       , block_out_channels[0]       , kernel_size=3       , padding=(1, 1))

      # time
      if time_embedding_type == "fourier":
         __snake_case					:   str  				=			GaussianFourierProjection(embedding_size=block_out_channels[0]       , scale=1_6)
         __snake_case					:   Tuple  				=			2 * block_out_channels[0]
      elif time_embedding_type == "positional":
         __snake_case					:   List[str]  				=			Timesteps(block_out_channels[0]       , __a       , __a)
         __snake_case					:   Optional[Any]  				=			block_out_channels[0]

      __snake_case					:   List[Any]  				=			TimestepEmbedding(__a       , __a)

      # class embedding
      if class_embed_type is None and num_class_embeds is not None:
         __snake_case					:   Optional[Any]  				=			nn.Embedding(__a       , __a)
      elif class_embed_type == "timestep":
         __snake_case					:   int  				=			TimestepEmbedding(__a       , __a)
      elif class_embed_type == "identity":
         __snake_case					:   Any  				=			nn.Identity(__a       , __a)
      else:
         __snake_case					:   Dict  				=			None

      __snake_case					:   Union[str, Any]  				=			nn.ModuleList([])
      __snake_case					:   Dict  				=			None
      __snake_case					:   Union[str, Any]  				=			nn.ModuleList([])

      # down
      __snake_case					:   List[str]  				=			block_out_channels[0]
      for i, down_block_type in enumerate(__a):
         __snake_case					:   List[str]  				=			output_channel
         __snake_case					:   Tuple  				=			block_out_channels[i]
         __snake_case					:   Union[str, Any]  				=			i == len(__a) - 1

         __snake_case					:   Dict  				=			get_down_block(
             __a       , num_layers=__a       , in_channels=__a       , out_channels=__a       , temb_channels=__a       , add_downsample=not is_final_block       , resnet_eps=__a       , resnet_act_fn=__a       , resnet_groups=__a       , attention_head_dim=attention_head_dim if attention_head_dim is not None else output_channel       , downsample_padding=__a       , resnet_time_scale_shift=__a       , downsample_type=__a       , )
         self.down_blocks.append(__a)

      # mid
      __snake_case					:   List[str]  				=			UNetMidBlockaD(
          in_channels=block_out_channels[-1]       , temb_channels=__a       , resnet_eps=__a       , resnet_act_fn=__a       , output_scale_factor=__a       , resnet_time_scale_shift=__a       , attention_head_dim=attention_head_dim if attention_head_dim is not None else block_out_channels[-1]       , resnet_groups=__a       , add_attention=__a       , )

      # up
      __snake_case					:   List[Any]  				=			list(reversed(__a))
      __snake_case					:   Any  				=			reversed_block_out_channels[0]
      for i, up_block_type in enumerate(__a):
         __snake_case					:   str  				=			output_channel
         __snake_case					:   List[Any]  				=			reversed_block_out_channels[i]
         __snake_case					:   Optional[Any]  				=			reversed_block_out_channels[min(i + 1       , len(__a) - 1)]

         __snake_case					:   Optional[int]  				=			i == len(__a) - 1

         __snake_case					:   List[Any]  				=			get_up_block(
             __a       , num_layers=layers_per_block + 1       , in_channels=__a       , out_channels=__a       , prev_output_channel=__a       , temb_channels=__a       , add_upsample=not is_final_block       , resnet_eps=__a       , resnet_act_fn=__a       , resnet_groups=__a       , attention_head_dim=attention_head_dim if attention_head_dim is not None else output_channel       , resnet_time_scale_shift=__a       , upsample_type=__a       , )
         self.up_blocks.append(__a)
         __snake_case					:   Dict  				=			output_channel

      # out
      __snake_case					:   Tuple  				=			norm_num_groups if norm_num_groups is not None else min(block_out_channels[0] // 4       , 3_2)
      __snake_case					:   Tuple  				=			nn.GroupNorm(num_channels=block_out_channels[0]       , num_groups=__a       , eps=__a)
      __snake_case					:   List[str]  				=			nn.SiLU()
      __snake_case					:   List[Any]  				=			nn.Convad(block_out_channels[0]       , __a       , kernel_size=3       , padding=1)







   def      SCREAMING_SNAKE_CASE__       (self       , __a       , __a       , __a = None       , __a = True       , )      ->   Union[UNetaDOutput, Tuple]:
      """simple docstring"""






      if self.config.center_input_sample:
         __snake_case					:   Optional[Any]  				=			2 * sample - 1.0

      # 1. time
      __snake_case					:   Any  				=			timestep
      if not torch.is_tensor(__a):
         __snake_case					:   Dict  				=			torch.tensor([timesteps]       , dtype=torch.long       , device=sample.device)
      elif torch.is_tensor(__a) and len(timesteps.shape) == 0:
         __snake_case					:   Dict  				=			timesteps[None].to(sample.device)

      # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
      __snake_case					:   Optional[int]  				=			timesteps * torch.ones(sample.shape[0]       , dtype=timesteps.dtype       , device=timesteps.device)

      __snake_case					:   Dict  				=			self.time_proj(__a)

      # timesteps does not contain any weights and will always return f32 tensors
      # but time_embedding might actually be running in fp16. so we need to cast here.
      # there might be better ways to encapsulate this.
      __snake_case					:   Optional[Any]  				=			t_emb.to(dtype=self.dtype)
      __snake_case					:   List[Any]  				=			self.time_embedding(__a)

      if self.class_embedding is not None:
         if class_labels is None:
            raise ValueError('class_labels should be provided when doing class conditioning')

         if self.config.class_embed_type == "timestep":
            __snake_case					:   str  				=			self.time_proj(__a)

         __snake_case					:   List[str]  				=			self.class_embedding(__a).to(dtype=self.dtype)
         __snake_case					:   Optional[Any]  				=			emb + class_emb

      # 2. pre-process
      __snake_case					:   List[str]  				=			sample
      __snake_case					:   Optional[Any]  				=			self.conv_in(__a)

      # 3. down
      __snake_case					:   List[str]  				=			(sample,)
      for downsample_block in self.down_blocks:
         if hasattr(__a       , 'skip_conv'):
            __snake_case				,__snake_case				,__snake_case					:   List[str]  				=			downsample_block(
                hidden_states=__a       , temb=__a       , skip_sample=__a)
         else:
            __snake_case				,__snake_case					:   Dict  				=			downsample_block(hidden_states=__a       , temb=__a)

         down_block_res_samples += res_samples

      # 4. mid
      __snake_case					:   Any  				=			self.mid_block(__a       , __a)

      # 5. up
      __snake_case					:   Dict  				=			None
      for upsample_block in self.up_blocks:
         __snake_case					:   int  				=			down_block_res_samples[-len(upsample_block.resnets) :]
         __snake_case					:   Union[str, Any]  				=			down_block_res_samples[: -len(upsample_block.resnets)]

         if hasattr(__a       , 'skip_conv'):
            __snake_case				,__snake_case					:   List[str]  				=			upsample_block(__a       , __a       , __a       , __a)
         else:
            __snake_case					:   int  				=			upsample_block(__a       , __a       , __a)

        # 6. post-process
      __snake_case					:   List[str]  				=			self.conv_norm_out(__a)
      __snake_case					:   Optional[int]  				=			self.conv_act(__a)
      __snake_case					:   Optional[int]  				=			self.conv_out(__a)

      if skip_sample is not None:
         sample += skip_sample

      if self.config.time_embedding_type == "fourier":
         __snake_case					:   str  				=			timesteps.reshape((sample.shape[0], *([1] * len(sample.shape[1:]))))
         __snake_case					:   Union[str, Any]  				=			sample / timesteps

      if not return_dict:
         return (sample,)

      return UNetaDOutput(sample=__a)







'''simple docstring'''






import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class a_     (     UpperCamelCase_		,   unittest.TestCase				):
   _snake_case	   =				ElectraTokenizer
   _snake_case	   =				ElectraTokenizerFast
   _snake_case	   =				True
   _snake_case	   =				True
   _snake_case	   =				filter_non_english







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Tuple:
      """simple docstring"""






      super().setUp()

      __snake_case					:   List[Any]  				=			[
          '[UNK]',
          '[CLS]',
          '[SEP]',
          '[PAD]',
          '[MASK]',
          'want',
          '##want',
          '##ed',
          'wa',
          'un',
          'runn',
          '##ing',
          ',',
          'low',
          'lowest',
      ]
      __snake_case					:   int  				=			os.path.join(self.tmpdirname       , VOCAB_FILES_NAMES['vocab_file'])
      with open(self.vocab_file       , 'w'       , encoding='utf-8') as vocab_writer:
         vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]))







   def      SCREAMING_SNAKE_CASE__       (self       , __a)      ->   int:
      """simple docstring"""






      __snake_case					:   Union[str, Any]  				=			'UNwant\u00E9d,running'
      __snake_case					:   Tuple  				=			'unwanted, running'
      return input_text, output_text







   def      SCREAMING_SNAKE_CASE__       (self)      ->   List[str]:
      """simple docstring"""






      __snake_case					:   Union[str, Any]  				=			self.tokenizer_class(self.vocab_file)

      __snake_case					:   str  				=			tokenizer.tokenize('UNwant\u00E9d,running')
      self.assertListEqual(__a       , ['un', '##want', '##ed', ',', 'runn', '##ing'])
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__a)       , [9, 6, 7, 1_2, 1_0, 1_1])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   List[str]:
      """simple docstring"""






      if not self.test_rust_tokenizer:
         return

      __snake_case					:   Union[str, Any]  				=			self.get_tokenizer()
      __snake_case					:   Dict  				=			self.get_rust_tokenizer()

      __snake_case					:   Dict  				=			'UNwant\u00E9d,running'

      __snake_case					:   Tuple  				=			tokenizer.tokenize(__a)
      __snake_case					:   List[Any]  				=			rust_tokenizer.tokenize(__a)
      self.assertListEqual(__a       , __a)

      __snake_case					:   Any  				=			tokenizer.encode(__a       , add_special_tokens=__a)
      __snake_case					:   int  				=			rust_tokenizer.encode(__a       , add_special_tokens=__a)
      self.assertListEqual(__a       , __a)

      __snake_case					:   List[str]  				=			self.get_rust_tokenizer()
      __snake_case					:   Optional[int]  				=			tokenizer.encode(__a)
      __snake_case					:   Dict  				=			rust_tokenizer.encode(__a)
      self.assertListEqual(__a       , __a)

      # With lower casing
      __snake_case					:   Union[str, Any]  				=			self.get_tokenizer(do_lower_case=__a)
      __snake_case					:   List[Any]  				=			self.get_rust_tokenizer(do_lower_case=__a)

      __snake_case					:   List[str]  				=			'UNwant\u00E9d,running'

      __snake_case					:   List[Any]  				=			tokenizer.tokenize(__a)
      __snake_case					:   Union[str, Any]  				=			rust_tokenizer.tokenize(__a)
      self.assertListEqual(__a       , __a)

      __snake_case					:   List[str]  				=			tokenizer.encode(__a       , add_special_tokens=__a)
      __snake_case					:   int  				=			rust_tokenizer.encode(__a       , add_special_tokens=__a)
      self.assertListEqual(__a       , __a)

      __snake_case					:   List[Any]  				=			self.get_rust_tokenizer()
      __snake_case					:   Optional[Any]  				=			tokenizer.encode(__a)
      __snake_case					:   Tuple  				=			rust_tokenizer.encode(__a)
      self.assertListEqual(__a       , __a)







   def      SCREAMING_SNAKE_CASE__       (self)      ->   int:
      """simple docstring"""






      __snake_case					:   int  				=			BasicTokenizer()

      self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz')       , ['ah', '\u535A', '\u63A8', 'zz'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Tuple:
      """simple docstring"""






      __snake_case					:   Optional[Any]  				=			BasicTokenizer(do_lower_case=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')       , ['hello', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')       , ['hello'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Tuple:
      """simple docstring"""






      __snake_case					:   Any  				=			BasicTokenizer(do_lower_case=__a       , strip_accents=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')       , ['hällo', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')       , ['h\u00E9llo'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   str:
      """simple docstring"""






      __snake_case					:   Optional[Any]  				=			BasicTokenizer(do_lower_case=__a       , strip_accents=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')       , ['hallo', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')       , ['hello'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   str:
      """simple docstring"""






      __snake_case					:   Tuple  				=			BasicTokenizer(do_lower_case=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')       , ['hallo', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')       , ['hello'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Union[str, Any]:
      """simple docstring"""






      __snake_case					:   Dict  				=			BasicTokenizer(do_lower_case=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')       , ['HeLLo', '!', 'how', 'Are', 'yoU', '?'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   int:
      """simple docstring"""






      __snake_case					:   Union[str, Any]  				=			BasicTokenizer(do_lower_case=__a       , strip_accents=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')       , ['HäLLo', '!', 'how', 'Are', 'yoU', '?'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   int:
      """simple docstring"""






      __snake_case					:   List[Any]  				=			BasicTokenizer(do_lower_case=__a       , strip_accents=__a)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')       , ['HaLLo', '!', 'how', 'Are', 'yoU', '?'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   str:
      """simple docstring"""






      __snake_case					:   Union[str, Any]  				=			BasicTokenizer(do_lower_case=__a       , never_split=['[UNK]'])

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]')       , ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   List[str]:
      """simple docstring"""






      __snake_case					:   Tuple  				=			['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

      __snake_case					:   List[str]  				=			{}
      for i, token in enumerate(__a):
         __snake_case					:   str  				=			i
      __snake_case					:   List[str]  				=			WordpieceTokenizer(vocab=__a       , unk_token='[UNK]')

      self.assertListEqual(tokenizer.tokenize('')       , [])

      self.assertListEqual(tokenizer.tokenize('unwanted running')       , ['un', '##want', '##ed', 'runn', '##ing'])

      self.assertListEqual(tokenizer.tokenize('unwantedX running')       , ['[UNK]', 'runn', '##ing'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Tuple:
      """simple docstring"""






      self.assertTrue(_is_whitespace(' '))
      self.assertTrue(_is_whitespace('\t'))
      self.assertTrue(_is_whitespace('\r'))
      self.assertTrue(_is_whitespace('\n'))
      self.assertTrue(_is_whitespace('\u00A0'))

      self.assertFalse(_is_whitespace('A'))
      self.assertFalse(_is_whitespace('-'))







   def      SCREAMING_SNAKE_CASE__       (self)      ->   List[str]:
      """simple docstring"""






      self.assertTrue(_is_control('\u0005'))

      self.assertFalse(_is_control('A'))
      self.assertFalse(_is_control(' '))
      self.assertFalse(_is_control('\t'))
      self.assertFalse(_is_control('\r'))







   def      SCREAMING_SNAKE_CASE__       (self)      ->   int:
      """simple docstring"""






      self.assertTrue(_is_punctuation('-'))
      self.assertTrue(_is_punctuation('$'))
      self.assertTrue(_is_punctuation('`'))
      self.assertTrue(_is_punctuation('.'))

      self.assertFalse(_is_punctuation('A'))
      self.assertFalse(_is_punctuation(' '))







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Optional[Any]:
      """simple docstring"""






      __snake_case					:   Dict  				=			self.get_tokenizer()
      __snake_case					:   Dict  				=			self.get_rust_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(__a) for t in ['Test', '\xad', 'test']]       , [['[UNK]'], [], ['[UNK]']])

      self.assertListEqual(
          [rust_tokenizer.tokenize(__a) for t in ['Test', '\xad', 'test']]       , [['[UNK]'], [], ['[UNK]']])







   @slow
   def      SCREAMING_SNAKE_CASE__       (self)      ->   Tuple:
      """simple docstring"""






      __snake_case					:   str  				=			self.tokenizer_class.from_pretrained('google/electra-base-discriminator')

      __snake_case					:   Optional[Any]  				=			tokenizer.encode('sequence builders'       , add_special_tokens=__a)
      __snake_case					:   List[Any]  				=			tokenizer.encode('multi-sequence build'       , add_special_tokens=__a)

      __snake_case					:   Tuple  				=			tokenizer.build_inputs_with_special_tokens(__a)
      __snake_case					:   int  				=			tokenizer.build_inputs_with_special_tokens(__a       , __a)

      assert encoded_sentence == [1_0_1] + text + [1_0_2]
      assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Optional[int]:
      """simple docstring"""






      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
            __snake_case					:   str  				=			self.rust_tokenizer_class.from_pretrained(__a       , **__a)

            __snake_case					:   List[Any]  				=			F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
            __snake_case					:   str  				=			tokenizer_r.encode_plus(
                __a       , return_attention_mask=__a       , return_token_type_ids=__a       , return_offsets_mapping=__a       , add_special_tokens=__a       , )

            __snake_case					:   int  				=			tokenizer_r.do_lower_case if hasattr(__a       , 'do_lower_case') else False
            __snake_case					:   Any  				=			(
                [
                    ((0, 0), tokenizer_r.cls_token),
                    ((0, 1), 'A'),
                    ((1, 2), ','),
                    ((3, 5), 'na'),
                    ((5, 6), '##ï'),
                    ((6, 8), '##ve'),
                    ((9, 1_5), tokenizer_r.mask_token),
                    ((1_6, 2_1), 'Allen'),
                    ((2_1, 2_3), '##NL'),
                    ((2_3, 2_4), '##P'),
                    ((2_5, 3_3), 'sentence'),
                    ((3_3, 3_4), '.'),
                    ((0, 0), tokenizer_r.sep_token),
                ]
                if not do_lower_case
                else [
                    ((0, 0), tokenizer_r.cls_token),
                    ((0, 1), 'a'),
                    ((1, 2), ','),
                    ((3, 8), 'naive'),
                    ((9, 1_5), tokenizer_r.mask_token),
                    ((1_6, 2_1), 'allen'),
                    ((2_1, 2_3), '##nl'),
                    ((2_3, 2_4), '##p'),
                    ((2_5, 3_3), 'sentence'),
                    ((3_3, 3_4), '.'),
                    ((0, 0), tokenizer_r.sep_token),
                ]
            )

            self.assertEqual(
                [e[1] for e in expected_results]       , tokenizer_r.convert_ids_to_tokens(tokens['input_ids']))
            self.assertEqual([e[0] for e in expected_results]       , tokens['offset_mapping'])







   def      SCREAMING_SNAKE_CASE__       (self)      ->   Any:
      """simple docstring"""






      __snake_case					:   str  				=			['的', '人', '有']
      __snake_case					:   str  				=			''.join(__a)
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
            __snake_case					:   Tuple  				=			True
            __snake_case					:   str  				=			self.tokenizer_class.from_pretrained(__a       , **__a)
            __snake_case					:   Optional[int]  				=			self.rust_tokenizer_class.from_pretrained(__a       , **__a)

            __snake_case					:   List[str]  				=			tokenizer_p.encode(__a       , add_special_tokens=__a)
            __snake_case					:   Optional[Any]  				=			tokenizer_r.encode(__a       , add_special_tokens=__a)

            __snake_case					:   Union[str, Any]  				=			tokenizer_r.convert_ids_to_tokens(__a)
            __snake_case					:   List[Any]  				=			tokenizer_p.convert_ids_to_tokens(__a)

            # it is expected that each Chinese character is not preceded by "##"
            self.assertListEqual(__a       , __a)
            self.assertListEqual(__a       , __a)

            __snake_case					:   Optional[int]  				=			False
            __snake_case					:   Any  				=			self.rust_tokenizer_class.from_pretrained(__a       , **__a)
            __snake_case					:   List[str]  				=			self.tokenizer_class.from_pretrained(__a       , **__a)

            __snake_case					:   List[str]  				=			tokenizer_r.encode(__a       , add_special_tokens=__a)
            __snake_case					:   Optional[Any]  				=			tokenizer_p.encode(__a       , add_special_tokens=__a)

            __snake_case					:   List[str]  				=			tokenizer_r.convert_ids_to_tokens(__a)
            __snake_case					:   Optional[int]  				=			tokenizer_p.convert_ids_to_tokens(__a)

            # it is expected that only the first Chinese character is not preceded by "##".
            __snake_case					:   Any  				=			[
                F"""##{token}""" if idx != 0 else token for idx, token in enumerate(__a)
            ]
            self.assertListEqual(__a       , __a)
            self.assertListEqual(__a       , __a)