'''simple docstring'''





import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


SCREAMING_SNAKE_CASE_:     Union[str, Any]  =logging.getLogger(__name__)





def 	lowerCAmelCase_				(					)    ->      Union[str, Any]:


     '''simple docstring'''




     UpperCAmelCase_           =   argparse.ArgumentParser(
         description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."			)
     parser.add_argument(
         "--dataset_name"					,    type=snake_case_					,    default="wikitext"					,    help="Name of the training. Explore datasets at: hf.co/datasets."					,    )
     parser.add_argument(
         "--dataset_config"					,    type=snake_case_					,    default="wikitext-103-raw-v1"					,    help="Configuration name of the dataset."			)
     parser.add_argument(
         "--tokenizer_name_or_path"					,    type=snake_case_					,    default="sayakpaul/unigram-tokenizer-wikitext"					,    help="Tokenizer identifier. Can be a local filepath or a Hub identifier."					,    )
     parser.add_argument(
         "--shard_size"					,    type=snake_case_					,    default=10_00					,    help="Number of entries to go in a single shard."					,    )
     parser.add_argument("--split"					,    type=snake_case_					,    default="train"					,    choices=["train", "test", "validation"]			)
     parser.add_argument(
         "--limit"					,    default=snake_case_					,    type=snake_case_					,    help="Limit the number of shards (used for debugging)."					,    )
     parser.add_argument(
         "--max_length"					,    type=snake_case_					,    default=5_12					,    help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
         " sequence length that is a multiple of 8."					,    )
     parser.add_argument(
         "--output_dir"					,    default="tf-tpu"					,    type=snake_case_					,    help="Output directory where the TFRecord shards will be saved. If the"
         " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
         " shards will be directly saved to a Google Cloud Storage bucket."					,    )

     UpperCAmelCase_           =   parser.parse_args()
     return args





def 	lowerCAmelCase_				(					snake_case_       : Dict			)    ->      str:


     '''simple docstring'''




     def fn(snake_case_       : Optional[Any]			):
          return tokenizer(examples["text"]			)

     return fn





def 	lowerCAmelCase_				(					snake_case_       : List[str]			)    ->      Union[str, Any]:


     '''simple docstring'''




     UpperCAmelCase_           =   []
     for i in range(len(tokenized_data["input_ids"]			)			):
          UpperCAmelCase_           =   {
              "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]			)			),
              "attention_mask": tf.train.Feature(
                  intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]			)			),
          }
          UpperCAmelCase_           =   tf.train.Features(feature=snake_case_			)
          UpperCAmelCase_           =   tf.train.Example(features=snake_case_			)
          UpperCAmelCase_           =   example.SerializeToString()
          records.append(snake_case_			)
     return records





def 	lowerCAmelCase_				(					snake_case_       : List[str]			)    ->      str:


     '''simple docstring'''




     UpperCAmelCase_           =   datasets.load_dataset(args.dataset_name					,    args.dataset_config					,    split=args.split			)

     if args.limit is not None:
          UpperCAmelCase_           =   min(len(snake_case_			)					,    args.limit			)
          UpperCAmelCase_           =   dataset.select(range(snake_case_			)			)
          print(f"""Limiting the dataset to {args.limit} entries."""			)

     UpperCAmelCase_           =   AutoTokenizer.from_pretrained(args.tokenizer_name_or_path			)

     # Handle output directory creation.
     # For serializing into a Google Cloud Storage Bucket, one needs to first
     # create a bucket.
     if "gs" not in args.output_dir:
          if not os.path.exists(args.output_dir			):
               os.makedirs(args.output_dir			)
          UpperCAmelCase_           =   os.path.join(args.output_dir					,    args.split			)
          if not os.path.exists(snake_case_			):
               os.makedirs(snake_case_			)
     else:
          UpperCAmelCase_           =   os.path.join(args.output_dir					,    args.split			)

     # Tokenize the whole dataset at once.
     UpperCAmelCase_           =   tokenize_function(snake_case_			)
     UpperCAmelCase_           =   dataset.map(snake_case_					,    batched=snake_case_					,    num_proc=4					,    remove_columns=["text"]			)

     # We need to concatenate all our texts together, and then split the result
     # into chunks of a fixed size, which we will call block_size. To do this, we
     # will use the map method again, with the option batched=True. When we use batched=True,
     # the function we pass to map() will be passed multiple inputs at once, allowing us
     # to group them into more or fewer examples than we had in the input.
     # This allows us to create our new fixed-length samples. The advantage of this
     # method is that we don't lose a whole lot of content from the dataset compared to the
     # case where we simply tokenize with a pre-defined max_length.

     def group_texts(snake_case_       : str			):
          # Concatenate all texts.
          UpperCAmelCase_           =   {k: sum(examples[k]					,    []			) for k in examples.keys()}
          UpperCAmelCase_           =   len(concatenated_examples[list(examples.keys()			)[0]]			)
          # We drop the small remainder, though you could add padding instead if the model supports it
          # In this, as in all things, we advise you to follow your heart 🫀
          UpperCAmelCase_           =   (total_length // args.max_length) * args.max_length
          # Split by chunks of max_len.
          UpperCAmelCase_           =   {
              k: [t[i : i + args.max_length] for i in range(0					,    snake_case_					,    args.max_length			)]
              for k, t in concatenated_examples.items()
          }
          return result

     UpperCAmelCase_           =   dataset_tokenized.map(snake_case_					,    batched=snake_case_					,    batch_size=10_00					,    num_proc=4			)

     UpperCAmelCase_           =   0
     UpperCAmelCase_           =   0
     for shard in range(0					,    len(snake_case_			)					,    args.shard_size			):
          UpperCAmelCase_           =   grouped_dataset[shard : shard + args.shard_size]
          UpperCAmelCase_           =   len(dataset_snapshot["input_ids"]			)
          UpperCAmelCase_           =   os.path.join(snake_case_					,    f"""dataset-{shard_count}-{records_containing}.tfrecord"""			)
          UpperCAmelCase_           =   get_serialized_examples(snake_case_			)

          with tf.io.TFRecordWriter(snake_case_			) as out_file:
               for i in range(len(snake_case_			)			):
                    UpperCAmelCase_           =   serialized_examples[i]
                    out_file.write(snake_case_			)
               print("Wrote file {} containing {} records".format(snake_case_					,    snake_case_			)			)

          shard_count += 1
          total_records += records_containing

     with open(f"""split-{args.split}-records-count.txt"""					,    "w"			) as f:
          print(f"""Total {args.split} records: {total_records}"""					,    file=snake_case_			)


if __name__ == "__main__":
    SCREAMING_SNAKE_CASE_:     List[str]  =parse_args()
    main(args)





'''simple docstring'''





import argparse
import io

import requests
import torch
from omegaconf import OmegaConf

from diffusers import AutoencoderKL
from diffusers.pipelines.stable_diffusion.convert_from_ckpt import (
    assign_to_checkpoint,
    conv_attn_to_linear,
    create_vae_diffusers_config,
    renew_vae_attention_paths,
    renew_vae_resnet_paths,
)





def 	lowerCAmelCase_				(					snake_case_       : List[Any]					,    snake_case_       : int			)    ->      Optional[int]:


     '''simple docstring'''




     UpperCAmelCase_           =   checkpoint

     UpperCAmelCase_           =   {}

     UpperCAmelCase_           =   vae_state_dict["encoder.conv_in.weight"]
     UpperCAmelCase_           =   vae_state_dict["encoder.conv_in.bias"]
     UpperCAmelCase_           =   vae_state_dict["encoder.conv_out.weight"]
     UpperCAmelCase_           =   vae_state_dict["encoder.conv_out.bias"]
     UpperCAmelCase_           =   vae_state_dict["encoder.norm_out.weight"]
     UpperCAmelCase_           =   vae_state_dict["encoder.norm_out.bias"]

     UpperCAmelCase_           =   vae_state_dict["decoder.conv_in.weight"]
     UpperCAmelCase_           =   vae_state_dict["decoder.conv_in.bias"]
     UpperCAmelCase_           =   vae_state_dict["decoder.conv_out.weight"]
     UpperCAmelCase_           =   vae_state_dict["decoder.conv_out.bias"]
     UpperCAmelCase_           =   vae_state_dict["decoder.norm_out.weight"]
     UpperCAmelCase_           =   vae_state_dict["decoder.norm_out.bias"]

     UpperCAmelCase_           =   vae_state_dict["quant_conv.weight"]
     UpperCAmelCase_           =   vae_state_dict["quant_conv.bias"]
     UpperCAmelCase_           =   vae_state_dict["post_quant_conv.weight"]
     UpperCAmelCase_           =   vae_state_dict["post_quant_conv.bias"]

     # Retrieves the keys for the encoder down blocks only
     UpperCAmelCase_           =   len({".".join(layer.split("."			)[:3]			) for layer in vae_state_dict if "encoder.down" in layer}			)
     UpperCAmelCase_           =   {
         layer_id: [key for key in vae_state_dict if f"""down.{layer_id}""" in key] for layer_id in range(snake_case_			)
     }

     # Retrieves the keys for the decoder up blocks only
     UpperCAmelCase_           =   len({".".join(layer.split("."			)[:3]			) for layer in vae_state_dict if "decoder.up" in layer}			)
     UpperCAmelCase_           =   {
         layer_id: [key for key in vae_state_dict if f"""up.{layer_id}""" in key] for layer_id in range(snake_case_			)
     }

     for i in range(snake_case_			):
          UpperCAmelCase_           =   [key for key in down_blocks[i] if f"""down.{i}""" in key and f"""down.{i}.downsample""" not in key]

          if f"""encoder.down.{i}.downsample.conv.weight""" in vae_state_dict:
               UpperCAmelCase_           =   vae_state_dict.pop(
                   f"""encoder.down.{i}.downsample.conv.weight"""			)
               UpperCAmelCase_           =   vae_state_dict.pop(
                   f"""encoder.down.{i}.downsample.conv.bias"""			)

          UpperCAmelCase_           =   renew_vae_resnet_paths(snake_case_			)
          UpperCAmelCase_           =   {"old": f"""down.{i}.block""", "new": f"""down_blocks.{i}.resnets"""}
          assign_to_checkpoint(snake_case_					,    snake_case_					,    snake_case_					,    additional_replacements=[meta_path]					,    config=snake_case_			)

     UpperCAmelCase_           =   [key for key in vae_state_dict if "encoder.mid.block" in key]
     UpperCAmelCase_           =   2
     for i in range(1					,    num_mid_res_blocks + 1			):
          UpperCAmelCase_           =   [key for key in mid_resnets if f"""encoder.mid.block_{i}""" in key]

          UpperCAmelCase_           =   renew_vae_resnet_paths(snake_case_			)
          UpperCAmelCase_           =   {"old": f"""mid.block_{i}""", "new": f"""mid_block.resnets.{i - 1}"""}
          assign_to_checkpoint(snake_case_					,    snake_case_					,    snake_case_					,    additional_replacements=[meta_path]					,    config=snake_case_			)

     UpperCAmelCase_           =   [key for key in vae_state_dict if "encoder.mid.attn" in key]
     UpperCAmelCase_           =   renew_vae_attention_paths(snake_case_			)
     UpperCAmelCase_           =   {"old": "mid.attn_1", "new": "mid_block.attentions.0"}
     assign_to_checkpoint(snake_case_					,    snake_case_					,    snake_case_					,    additional_replacements=[meta_path]					,    config=snake_case_			)
     conv_attn_to_linear(snake_case_			)

     for i in range(snake_case_			):
          UpperCAmelCase_           =   num_up_blocks - 1 - i
          UpperCAmelCase_           =   [
              key for key in up_blocks[block_id] if f"""up.{block_id}""" in key and f"""up.{block_id}.upsample""" not in key
          ]

          if f"""decoder.up.{block_id}.upsample.conv.weight""" in vae_state_dict:
               UpperCAmelCase_           =   vae_state_dict[
                   f"""decoder.up.{block_id}.upsample.conv.weight"""
               ]
               UpperCAmelCase_           =   vae_state_dict[
                   f"""decoder.up.{block_id}.upsample.conv.bias"""
               ]

          UpperCAmelCase_           =   renew_vae_resnet_paths(snake_case_			)
          UpperCAmelCase_           =   {"old": f"""up.{block_id}.block""", "new": f"""up_blocks.{i}.resnets"""}
          assign_to_checkpoint(snake_case_					,    snake_case_					,    snake_case_					,    additional_replacements=[meta_path]					,    config=snake_case_			)

     UpperCAmelCase_           =   [key for key in vae_state_dict if "decoder.mid.block" in key]
     UpperCAmelCase_           =   2
     for i in range(1					,    num_mid_res_blocks + 1			):
          UpperCAmelCase_           =   [key for key in mid_resnets if f"""decoder.mid.block_{i}""" in key]

          UpperCAmelCase_           =   renew_vae_resnet_paths(snake_case_			)
          UpperCAmelCase_           =   {"old": f"""mid.block_{i}""", "new": f"""mid_block.resnets.{i - 1}"""}
          assign_to_checkpoint(snake_case_					,    snake_case_					,    snake_case_					,    additional_replacements=[meta_path]					,    config=snake_case_			)

     UpperCAmelCase_           =   [key for key in vae_state_dict if "decoder.mid.attn" in key]
     UpperCAmelCase_           =   renew_vae_attention_paths(snake_case_			)
     UpperCAmelCase_           =   {"old": "mid.attn_1", "new": "mid_block.attentions.0"}
     assign_to_checkpoint(snake_case_					,    snake_case_					,    snake_case_					,    additional_replacements=[meta_path]					,    config=snake_case_			)
     conv_attn_to_linear(snake_case_			)
     return new_checkpoint





def 	lowerCAmelCase_				(					snake_case_       : str					,    snake_case_       : str					,    )    ->      Dict:


     '''simple docstring'''




     UpperCAmelCase_           =   requests.get(
         " https://raw.githubusercontent.com/CompVis/stable-diffusion/main/configs/stable-diffusion/v1-inference.yaml"			)
     UpperCAmelCase_           =   io.BytesIO(r.content			)

     UpperCAmelCase_           =   OmegaConf.load(snake_case_			)
     UpperCAmelCase_           =   5_12
     UpperCAmelCase_           =   "cuda" if torch.cuda.is_available() else "cpu"
     if checkpoint_path.endswith("safetensors"			):
          from safetensors import safe_open

          UpperCAmelCase_           =   {}
          with safe_open(snake_case_					,    framework="pt"					,    device="cpu"			) as f:
               for key in f.keys():
                    UpperCAmelCase_           =   f.get_tensor(snake_case_			)
     else:
          UpperCAmelCase_           =   torch.load(snake_case_					,    map_location=snake_case_			)["state_dict"]

     # Convert the VAE model.
     UpperCAmelCase_           =   create_vae_diffusers_config(snake_case_					,    image_size=snake_case_			)
     UpperCAmelCase_           =   custom_convert_ldm_vae_checkpoint(snake_case_					,    snake_case_			)

     UpperCAmelCase_           =   AutoencoderKL(**snake_case_			)
     vae.load_state_dict(snake_case_			)
     vae.save_pretrained(snake_case_			)


if __name__ == "__main__":
    SCREAMING_SNAKE_CASE_:     Optional[int]  =argparse.ArgumentParser()

    parser.add_argument('--vae_pt_path', default=None, type=str, required=True, help='Path to the VAE.pt to convert.')
    parser.add_argument('--dump_path', default=None, type=str, required=True, help='Path to the VAE.pt to convert.')

    SCREAMING_SNAKE_CASE_:     str  =parser.parse_args()

    vae_pt_to_vae_diffuser(args.vae_pt_path, args.dump_path)




