
"""simple docstring"""






import argparse
import json

import torch

from diffusers import AutoencoderKL, DDPMPipeline, DDPMScheduler, UNetaDModel, VQModel
def    UpperCamelCase						(					UpperCAmelCase ,   UpperCAmelCase=1     )						->List[str]:

   """simple docstring"""



   if n_shave_prefix_segments >= 0:
      return ".".join(path.split("."     )[n_shave_prefix_segments:]     )
   else:
      return ".".join(path.split("."     )[:n_shave_prefix_segments]     )
def    UpperCamelCase						(					UpperCAmelCase ,   UpperCAmelCase=0     )						->List[Any]:

   """simple docstring"""



   a_						   =     []
   for old_item in old_list:
      a_						   =     old_item
      a_						   =     new_item.replace("block." ,   "resnets."     )
      a_						   =     new_item.replace("conv_shorcut" ,   "conv1"     )
      a_						   =     new_item.replace("in_shortcut" ,   "conv_shortcut"     )
      a_						   =     new_item.replace("temb_proj" ,   "time_emb_proj"     )

      a_						   =     shave_segments(UpperCAmelCase ,   n_shave_prefix_segments=UpperCAmelCase     )

      mapping.append({"old": old_item, "new": new_item}     )

   return mapping
def    UpperCamelCase						(					UpperCAmelCase ,   UpperCAmelCase=0 ,   UpperCAmelCase=False     )						->Any:

   """simple docstring"""



   a_						   =     []
   for old_item in old_list:
      a_						   =     old_item

      # In `model.mid`, the layer is called `attn`.
      if not in_mid:
         a_						   =     new_item.replace("attn" ,   "attentions"     )
      a_						   =     new_item.replace(".k." ,   ".key."     )
      a_						   =     new_item.replace(".v." ,   ".value."     )
      a_						   =     new_item.replace(".q." ,   ".query."     )

      a_						   =     new_item.replace("proj_out" ,   "proj_attn"     )
      a_						   =     new_item.replace("norm" ,   "group_norm"     )

      a_						   =     shave_segments(UpperCAmelCase ,   n_shave_prefix_segments=UpperCAmelCase     )
      mapping.append({"old": old_item, "new": new_item}     )

   return mapping
def    UpperCamelCase						(					UpperCAmelCase ,   UpperCAmelCase ,   UpperCAmelCase ,   UpperCAmelCase=None ,   UpperCAmelCase=None ,   UpperCAmelCase=None     )						->Any:

   """simple docstring"""



   assert isinstance(UpperCAmelCase ,   UpperCAmelCase     ), "Paths should be a list of dicts containing 'old' and 'new' keys."

   if attention_paths_to_split is not None:
      if config is None:
         raise ValueError("Please specify the config if setting 'attention_paths_to_split' to 'True'."     )

      for path, path_map in attention_paths_to_split.items():
         a_						   =     old_checkpoint[path]
         a_						   =     old_tensor.shape[0] // 3

         a_						   =     (-1, channels) if len(old_tensor.shape     ) == 3 else (-1)

         a_						   =     old_tensor.shape[0] // config.get("num_head_channels" ,   1     ) // 3

         a_						   =     old_tensor.reshape((num_heads, 3 * channels // num_heads) + old_tensor.shape[1:]     )
         a_   ,			a_   ,			a_						   =     old_tensor.split(channels // num_heads ,   dim=1     )

         a_						   =     query.reshape(UpperCAmelCase     ).squeeze()
         a_						   =     key.reshape(UpperCAmelCase     ).squeeze()
         a_						   =     value.reshape(UpperCAmelCase     ).squeeze()

   for path in paths:
      a_						   =     path["new"]

      if attention_paths_to_split is not None and new_path in attention_paths_to_split:
         continue

      a_						   =     new_path.replace("down." ,   "down_blocks."     )
      a_						   =     new_path.replace("up." ,   "up_blocks."     )

      if additional_replacements is not None:
         for replacement in additional_replacements:
            a_						   =     new_path.replace(replacement["old"] ,   replacement["new"]     )

      if "attentions" in new_path:
         a_						   =     old_checkpoint[path["old"]].squeeze()
      else:
         a_						   =     old_checkpoint[path["old"]]
def    UpperCamelCase						(					UpperCAmelCase ,   UpperCAmelCase     )						->List[str]:

   """simple docstring"""



   a_						   =     {}

   a_						   =     checkpoint["temb.dense.0.weight"]
   a_						   =     checkpoint["temb.dense.0.bias"]
   a_						   =     checkpoint["temb.dense.1.weight"]
   a_						   =     checkpoint["temb.dense.1.bias"]

   a_						   =     checkpoint["norm_out.weight"]
   a_						   =     checkpoint["norm_out.bias"]

   a_						   =     checkpoint["conv_in.weight"]
   a_						   =     checkpoint["conv_in.bias"]
   a_						   =     checkpoint["conv_out.weight"]
   a_						   =     checkpoint["conv_out.bias"]

   a_						   =     len({".".join(layer.split("."     )[:2]     ) for layer in checkpoint if "down" in layer}     )
   a_						   =     {
       layer_id: [key for key in checkpoint if F'''down.{layer_id}''' in key] for layer_id in range(UpperCAmelCase     )
   }

   a_						   =     len({".".join(layer.split("."     )[:2]     ) for layer in checkpoint if "up" in layer}     )
   a_						   =     {layer_id: [key for key in checkpoint if F'''up.{layer_id}''' in key] for layer_id in range(UpperCAmelCase     )}

   for i in range(UpperCAmelCase     ):
      a_						   =     (i - 1) // (config["layers_per_block"] + 1)

      if any("downsample" in layer for layer in down_blocks[i]     ):
         a_						   =     checkpoint[
             F'''down.{i}.downsample.op.weight'''
         ]
         a_						   =     checkpoint[F'''down.{i}.downsample.op.bias''']
      #            new_checkpoint[f'down_blocks.{i}.downsamplers.0.op.weight'] = checkpoint[f'down.{i}.downsample.conv.weight']
      #            new_checkpoint[f'down_blocks.{i}.downsamplers.0.op.bias'] = checkpoint[f'down.{i}.downsample.conv.bias']

      if any("block" in layer for layer in down_blocks[i]     ):
         a_						   =     len(
             {".".join(shave_segments(UpperCAmelCase ,   2     ).split("."     )[:2]     ) for layer in down_blocks[i] if "block" in layer}     )
         a_						   =     {
             layer_id: [key for key in down_blocks[i] if F'''block.{layer_id}''' in key]
             for layer_id in range(UpperCAmelCase     )
         }

         if num_blocks > 0:
            for j in range(config["layers_per_block"]     ):
               a_						   =     renew_resnet_paths(blocks[j]     )
               assign_to_checkpoint(UpperCAmelCase ,   UpperCAmelCase ,   UpperCAmelCase     )

      if any("attn" in layer for layer in down_blocks[i]     ):
         a_						   =     len(
             {".".join(shave_segments(UpperCAmelCase ,   2     ).split("."     )[:2]     ) for layer in down_blocks[i] if "attn" in layer}     )
         a_						   =     {
             layer_id: [key for key in down_blocks[i] if F'''attn.{layer_id}''' in key]
             for layer_id in range(UpperCAmelCase     )
         }

         if num_attn > 0:
            for j in range(config["layers_per_block"]     ):
               a_						   =     renew_attention_paths(attns[j]     )
               assign_to_checkpoint(UpperCAmelCase ,   UpperCAmelCase ,   UpperCAmelCase ,   config=UpperCAmelCase     )

   a_						   =     [key for key in checkpoint if "mid.block_1" in key]
   a_						   =     [key for key in checkpoint if "mid.block_2" in key]
   a_						   =     [key for key in checkpoint if "mid.attn_1" in key]

   # Mid new 2
   a_						   =     renew_resnet_paths(UpperCAmelCase     )
   assign_to_checkpoint(
       UpperCAmelCase ,   UpperCAmelCase ,   UpperCAmelCase ,   additional_replacements=[{"old": "mid.", "new": "mid_new_2."}, {"old": "block_1", "new": "resnets.0"}] ,   )

   a_						   =     renew_resnet_paths(UpperCAmelCase     )
   assign_to_checkpoint(
       UpperCAmelCase ,   UpperCAmelCase ,   UpperCAmelCase ,   additional_replacements=[{"old": "mid.", "new": "mid_new_2."}, {"old": "block_2", "new": "resnets.1"}] ,   )

   a_						   =     renew_attention_paths(UpperCAmelCase ,   in_mid=UpperCAmelCase     )
   assign_to_checkpoint(
       UpperCAmelCase ,   UpperCAmelCase ,   UpperCAmelCase ,   additional_replacements=[{"old": "mid.", "new": "mid_new_2."}, {"old": "attn_1", "new": "attentions.0"}] ,   )

   for i in range(UpperCAmelCase     ):
      a_						   =     num_up_blocks - 1 - i

      if any("upsample" in layer for layer in up_blocks[i]     ):
         a_						   =     checkpoint[
             F'''up.{i}.upsample.conv.weight'''
         ]
         a_						   =     checkpoint[F'''up.{i}.upsample.conv.bias''']

      if any("block" in layer for layer in up_blocks[i]     ):
         a_						   =     len(
             {".".join(shave_segments(UpperCAmelCase ,   2     ).split("."     )[:2]     ) for layer in up_blocks[i] if "block" in layer}     )
         a_						   =     {
             layer_id: [key for key in up_blocks[i] if F'''block.{layer_id}''' in key] for layer_id in range(UpperCAmelCase     )
         }

         if num_blocks > 0:
            for j in range(config["layers_per_block"] + 1     ):
               a_						   =     {"old": F'''up_blocks.{i}''', "new": F'''up_blocks.{block_id}'''}
               a_						   =     renew_resnet_paths(blocks[j]     )
               assign_to_checkpoint(UpperCAmelCase ,   UpperCAmelCase ,   UpperCAmelCase ,   additional_replacements=[replace_indices]     )

      if any("attn" in layer for layer in up_blocks[i]     ):
         a_						   =     len(
             {".".join(shave_segments(UpperCAmelCase ,   2     ).split("."     )[:2]     ) for layer in up_blocks[i] if "attn" in layer}     )
         a_						   =     {
             layer_id: [key for key in up_blocks[i] if F'''attn.{layer_id}''' in key] for layer_id in range(UpperCAmelCase     )
         }

         if num_attn > 0:
            for j in range(config["layers_per_block"] + 1     ):
               a_						   =     {"old": F'''up_blocks.{i}''', "new": F'''up_blocks.{block_id}'''}
               a_						   =     renew_attention_paths(attns[j]     )
               assign_to_checkpoint(UpperCAmelCase ,   UpperCAmelCase ,   UpperCAmelCase ,   additional_replacements=[replace_indices]     )

   a_						   =     {k.replace("mid_new_2" ,   "mid_block"     ): v for k, v in new_checkpoint.items()}
   return new_checkpoint
def    UpperCamelCase						(					UpperCAmelCase ,   UpperCAmelCase     )						->Dict:

   """simple docstring"""



   a_						   =     {}

   a_						   =     checkpoint["encoder.norm_out.weight"]
   a_						   =     checkpoint["encoder.norm_out.bias"]

   a_						   =     checkpoint["encoder.conv_in.weight"]
   a_						   =     checkpoint["encoder.conv_in.bias"]
   a_						   =     checkpoint["encoder.conv_out.weight"]
   a_						   =     checkpoint["encoder.conv_out.bias"]

   a_						   =     checkpoint["decoder.norm_out.weight"]
   a_						   =     checkpoint["decoder.norm_out.bias"]

   a_						   =     checkpoint["decoder.conv_in.weight"]
   a_						   =     checkpoint["decoder.conv_in.bias"]
   a_						   =     checkpoint["decoder.conv_out.weight"]
   a_						   =     checkpoint["decoder.conv_out.bias"]

   a_						   =     len({".".join(layer.split("."     )[:3]     ) for layer in checkpoint if "down" in layer}     )
   a_						   =     {
       layer_id: [key for key in checkpoint if F'''down.{layer_id}''' in key] for layer_id in range(UpperCAmelCase     )
   }

   a_						   =     len({".".join(layer.split("."     )[:3]     ) for layer in checkpoint if "up" in layer}     )
   a_						   =     {layer_id: [key for key in checkpoint if F'''up.{layer_id}''' in key] for layer_id in range(UpperCAmelCase     )}

   for i in range(UpperCAmelCase     ):
      a_						   =     (i - 1) // (config["layers_per_block"] + 1)

      if any("downsample" in layer for layer in down_blocks[i]     ):
         a_						   =     checkpoint[
             F'''encoder.down.{i}.downsample.conv.weight'''
         ]
         a_						   =     checkpoint[
             F'''encoder.down.{i}.downsample.conv.bias'''
         ]

      if any("block" in layer for layer in down_blocks[i]     ):
         a_						   =     len(
             {".".join(shave_segments(UpperCAmelCase ,   3     ).split("."     )[:3]     ) for layer in down_blocks[i] if "block" in layer}     )
         a_						   =     {
             layer_id: [key for key in down_blocks[i] if F'''block.{layer_id}''' in key]
             for layer_id in range(UpperCAmelCase     )
         }

         if num_blocks > 0:
            for j in range(config["layers_per_block"]     ):
               a_						   =     renew_resnet_paths(blocks[j]     )
               assign_to_checkpoint(UpperCAmelCase ,   UpperCAmelCase ,   UpperCAmelCase     )

      if any("attn" in layer for layer in down_blocks[i]     ):
         a_						   =     len(
             {".".join(shave_segments(UpperCAmelCase ,   3     ).split("."     )[:3]     ) for layer in down_blocks[i] if "attn" in layer}     )
         a_						   =     {
             layer_id: [key for key in down_blocks[i] if F'''attn.{layer_id}''' in key]
             for layer_id in range(UpperCAmelCase     )
         }

         if num_attn > 0:
            for j in range(config["layers_per_block"]     ):
               a_						   =     renew_attention_paths(attns[j]     )
               assign_to_checkpoint(UpperCAmelCase ,   UpperCAmelCase ,   UpperCAmelCase ,   config=UpperCAmelCase     )

   a_						   =     [key for key in checkpoint if "mid.block_1" in key]
   a_						   =     [key for key in checkpoint if "mid.block_2" in key]
   a_						   =     [key for key in checkpoint if "mid.attn_1" in key]

   # Mid new 2
   a_						   =     renew_resnet_paths(UpperCAmelCase     )
   assign_to_checkpoint(
       UpperCAmelCase ,   UpperCAmelCase ,   UpperCAmelCase ,   additional_replacements=[{"old": "mid.", "new": "mid_new_2."}, {"old": "block_1", "new": "resnets.0"}] ,   )

   a_						   =     renew_resnet_paths(UpperCAmelCase     )
   assign_to_checkpoint(
       UpperCAmelCase ,   UpperCAmelCase ,   UpperCAmelCase ,   additional_replacements=[{"old": "mid.", "new": "mid_new_2."}, {"old": "block_2", "new": "resnets.1"}] ,   )

   a_						   =     renew_attention_paths(UpperCAmelCase ,   in_mid=UpperCAmelCase     )
   assign_to_checkpoint(
       UpperCAmelCase ,   UpperCAmelCase ,   UpperCAmelCase ,   additional_replacements=[{"old": "mid.", "new": "mid_new_2."}, {"old": "attn_1", "new": "attentions.0"}] ,   )

   for i in range(UpperCAmelCase     ):
      a_						   =     num_up_blocks - 1 - i

      if any("upsample" in layer for layer in up_blocks[i]     ):
         a_						   =     checkpoint[
             F'''decoder.up.{i}.upsample.conv.weight'''
         ]
         a_						   =     checkpoint[
             F'''decoder.up.{i}.upsample.conv.bias'''
         ]

      if any("block" in layer for layer in up_blocks[i]     ):
         a_						   =     len(
             {".".join(shave_segments(UpperCAmelCase ,   3     ).split("."     )[:3]     ) for layer in up_blocks[i] if "block" in layer}     )
         a_						   =     {
             layer_id: [key for key in up_blocks[i] if F'''block.{layer_id}''' in key] for layer_id in range(UpperCAmelCase     )
         }

         if num_blocks > 0:
            for j in range(config["layers_per_block"] + 1     ):
               a_						   =     {"old": F'''up_blocks.{i}''', "new": F'''up_blocks.{block_id}'''}
               a_						   =     renew_resnet_paths(blocks[j]     )
               assign_to_checkpoint(UpperCAmelCase ,   UpperCAmelCase ,   UpperCAmelCase ,   additional_replacements=[replace_indices]     )

      if any("attn" in layer for layer in up_blocks[i]     ):
         a_						   =     len(
             {".".join(shave_segments(UpperCAmelCase ,   3     ).split("."     )[:3]     ) for layer in up_blocks[i] if "attn" in layer}     )
         a_						   =     {
             layer_id: [key for key in up_blocks[i] if F'''attn.{layer_id}''' in key] for layer_id in range(UpperCAmelCase     )
         }

         if num_attn > 0:
            for j in range(config["layers_per_block"] + 1     ):
               a_						   =     {"old": F'''up_blocks.{i}''', "new": F'''up_blocks.{block_id}'''}
               a_						   =     renew_attention_paths(attns[j]     )
               assign_to_checkpoint(UpperCAmelCase ,   UpperCAmelCase ,   UpperCAmelCase ,   additional_replacements=[replace_indices]     )

   a_						   =     {k.replace("mid_new_2" ,   "mid_block"     ): v for k, v in new_checkpoint.items()}
   a_						   =     checkpoint["quant_conv.weight"]
   a_						   =     checkpoint["quant_conv.bias"]
   if "quantize.embedding.weight" in checkpoint:
      a_						   =     checkpoint["quantize.embedding.weight"]
   a_						   =     checkpoint["post_quant_conv.weight"]
   a_						   =     checkpoint["post_quant_conv.bias"]

   return new_checkpoint


if __name__ == "__main__":
      UpperCamelCase_						   =   argparse.ArgumentParser()

      parser.add_argument(
          '--checkpoint_path', default=None, type=str, required=True, help='Path to the checkpoint to convert.'
      )

      parser.add_argument(
          '--config_file',
          default=None,
          type=str,
          required=True,
          help='The config json file corresponding to the architecture.',
      )

      parser.add_argument('--dump_path', default=None, type=str, required=True, help='Path to the output model.')

      UpperCamelCase_						   =   parser.parse_args()
      UpperCamelCase_						   =   torch.load(args.checkpoint_path)

      with open(args.config_file) as f:
            UpperCamelCase_						   =   json.loads(f.read())

      # unet case
      UpperCamelCase_						   =   {key.split('.')[0] for key in checkpoint.keys()}
      if "encoder" in key_prefix_set and "decoder" in key_prefix_set:
            UpperCamelCase_						   =   convert_vq_autoenc_checkpoint(checkpoint, config)
      else:
            UpperCamelCase_						   =   convert_ddpm_checkpoint(checkpoint, config)

      if "ddpm" in config:
            del config["ddpm"]

      if config["_class_name"] == "VQModel":
            UpperCamelCase_						   =   VQModel(**config)
            model.load_state_dict(converted_checkpoint)
            model.save_pretrained(args.dump_path)
      elif config["_class_name"] == "AutoencoderKL":
            UpperCamelCase_						   =   AutoencoderKL(**config)
            model.load_state_dict(converted_checkpoint)
            model.save_pretrained(args.dump_path)
      else:
            UpperCamelCase_						   =   UNetaDModel(**config)
            model.load_state_dict(converted_checkpoint)

            UpperCamelCase_						   =   DDPMScheduler.from_config('/'.join(args.checkpoint_path.split('/')[:-1]))

            UpperCamelCase_						   =   DDPMPipeline(unet=model, scheduler=scheduler)
            pipe.save_pretrained(args.dump_path)

"""simple docstring"""






import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 							snake_case       (     SCREAMING_SNAKE_CASE_							,       unittest.TestCase		):
   a_					:		Optional[int]     					=     GPTNeoXJapaneseTokenizer
   a_					:		Dict     					=     False
   a_					:		Union[str, Any]     					=     {"""do_clean_text""": False, """add_prefix_space""": False}







   def    UpperCAmelCase__	( self)   ->Union[str, Any]:
      super().setUp()

      a_						   =     [
          "こん",
          "こんに",
          "にちは",
          "ばんは",
          "世界,㔺界",
          "、",
          "。",
          "<BR>",
          "<SP>",
          "<TAB>",
          "<URL>",
          "<EMAIL>",
          "<TEL>",
          "<DATE>",
          "<PRICE>",
          "<BLOCK>",
          "<KIGOU>",
          "<U2000U2BFF>",
          "<|emoji1|>",
          "<unk>",
          "<|startoftext|>",
          "<|endoftext|>",
      ]
      a_						   =     {"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
      a_						   =     {"unk_token": "<unk>"}

      a_						   =     os.path.join(self.tmpdirname       ,							VOCAB_FILES_NAMES["vocab_file"])
      a_						   =     os.path.join(self.tmpdirname       ,							VOCAB_FILES_NAMES["emoji_file"])
      with open(self.vocab_file       ,							"w"       ,							encoding="utf-8") as vocab_writer:
         vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
      with open(self.emoji_file       ,							"w") as emoji_writer:
         emoji_writer.write(json.dumps(__UpperCAmelCase))







   def    UpperCAmelCase__	( self       ,							**__UpperCAmelCase)   ->int:
      kwargs.update(self.special_tokens_map)
      return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname       ,							**__UpperCAmelCase)







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Dict:
      a_						   =     "こんにちは、世界。 \nこんばんは、㔺界。😀"
      a_						   =     "こんにちは、世界。 \nこんばんは、世界。😀"
      return input_text, output_text







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Optional[Any]:
      a_   ,			a_						   =     self.get_input_output_texts(__UpperCAmelCase)
      a_						   =     tokenizer.encode(__UpperCAmelCase       ,							add_special_tokens=__UpperCAmelCase)
      a_						   =     tokenizer.decode(__UpperCAmelCase       ,							clean_up_tokenization_spaces=__UpperCAmelCase)
      return text, ids







   def    UpperCAmelCase__	( self)   ->Any:
      pass  # TODO add if relevant







   def    UpperCAmelCase__	( self)   ->int:
      pass  # TODO add if relevant







   def    UpperCAmelCase__	( self)   ->Optional[Any]:
      pass  # TODO add if relevant







   def    UpperCAmelCase__	( self)   ->List[str]:
      a_						   =     self.get_tokenizer()

      # Testing tokenization
      a_						   =     "こんにちは、世界。　こんばんは、㔺界。"
      a_						   =     ["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
      a_						   =     tokenizer.tokenize(__UpperCAmelCase)
      self.assertListEqual(__UpperCAmelCase       ,							__UpperCAmelCase)

      # Testing conversion to ids without special tokens
      a_						   =     [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
      a_						   =     tokenizer.convert_tokens_to_ids(__UpperCAmelCase)
      self.assertListEqual(__UpperCAmelCase       ,							__UpperCAmelCase)

      # Testing conversion to ids with special tokens
      a_						   =     tokens + [tokenizer.unk_token]
      a_						   =     [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
      a_						   =     tokenizer.convert_tokens_to_ids(__UpperCAmelCase)
      self.assertListEqual(__UpperCAmelCase       ,							__UpperCAmelCase)







   @slow
   def    UpperCAmelCase__	( self)   ->Tuple:
      a_						   =     self.tokenizer_class.from_pretrained("abeja/gpt-neox-japanese-2.7b")

      a_						   =     tokenizer.encode("ありがとう。"       ,							add_special_tokens=__UpperCAmelCase)
      a_						   =     tokenizer.encode("どういたしまして。"       ,							add_special_tokens=__UpperCAmelCase)

      a_						   =     tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase)
      a_						   =     tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase       ,							__UpperCAmelCase)

      assert encoded_sentence == ids_a
      assert encoded_pair == ids_a + ids_a







   def    UpperCAmelCase__	( self)   ->Tuple:
      # Intentionally convert some words to accommodate character fluctuations unique to Japanese
      pass







   def    UpperCAmelCase__	( self)   ->Union[str, Any]:
      # tokenizer has no padding token
      pass