


'''simple docstring'''




import gc
import unittest

import numpy as np
import torch

from diffusers import StableDiffusionKDiffusionPipeline
from diffusers.utils import slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu


enable_full_determinism()

@slow
@require_torch_gpu
class 		UpperCAmelCase_      (  unittest.TestCase		):





     def 							__UpperCAmelCase      ( self       :      Optional[int]    )				->  int:
        # clean up the VRAM after each test
        super().tearDown()
        gc.collect()
        torch.cuda.empty_cache()





     def 							__UpperCAmelCase      ( self       :      Tuple    )				->  Any:
        lowerCAmelCase     =			StableDiffusionKDiffusionPipeline.from_pretrained('CompVis/stable-diffusion-v1-4'    )
        lowerCAmelCase     =			sd_pipe.to(UpperCAmelCase__    )
        sd_pipe.set_progress_bar_config(disable=UpperCAmelCase__    )

        sd_pipe.set_scheduler('sample_euler'    )

        lowerCAmelCase     =			'A painting of a squirrel eating a burger'
        lowerCAmelCase     =			torch.manual_seed(0    )
        lowerCAmelCase     =			sd_pipe([prompt]    ,      generator=UpperCAmelCase__    ,      guidance_scale=9.0    ,      num_inference_steps=2_0    ,      output_type='np'    )

        lowerCAmelCase     =			output.images

        lowerCAmelCase     =			image[0, -3:, -3:, -1]

        assert image.shape == (1, 5_1_2, 5_1_2, 3)
        lowerCAmelCase     =			np.array([0.0_447, 0.0_492, 0.0_468, 0.0_408, 0.0_383, 0.0_408, 0.0_354, 0.0_380, 0.0_339]    )

        assert np.abs(image_slice.flatten() - expected_slice    ).max() < 1E-2





     def 							__UpperCAmelCase      ( self       :      List[str]    )				->  Dict:
        lowerCAmelCase     =			StableDiffusionKDiffusionPipeline.from_pretrained('stabilityai/stable-diffusion-2-1-base'    )
        lowerCAmelCase     =			sd_pipe.to(UpperCAmelCase__    )
        sd_pipe.set_progress_bar_config(disable=UpperCAmelCase__    )

        sd_pipe.set_scheduler('sample_euler'    )

        lowerCAmelCase     =			'A painting of a squirrel eating a burger'
        lowerCAmelCase     =			torch.manual_seed(0    )
        lowerCAmelCase     =			sd_pipe([prompt]    ,      generator=UpperCAmelCase__    ,      guidance_scale=9.0    ,      num_inference_steps=2_0    ,      output_type='np'    )

        lowerCAmelCase     =			output.images

        lowerCAmelCase     =			image[0, -3:, -3:, -1]

        assert image.shape == (1, 5_1_2, 5_1_2, 3)
        lowerCAmelCase     =			np.array([0.1_237, 0.1_320, 0.1_438, 0.1_359, 0.1_390, 0.1_132, 0.1_277, 0.1_175, 0.1_112]    )

        assert np.abs(image_slice.flatten() - expected_slice    ).max() < 5E-1





     def 							__UpperCAmelCase      ( self       :      Optional[Any]    )				->  List[str]:
        lowerCAmelCase     =			StableDiffusionKDiffusionPipeline.from_pretrained('stabilityai/stable-diffusion-2-1-base'    )
        lowerCAmelCase     =			sd_pipe.to(UpperCAmelCase__    )
        sd_pipe.set_progress_bar_config(disable=UpperCAmelCase__    )

        sd_pipe.set_scheduler('sample_dpmpp_2m'    )

        lowerCAmelCase     =			'A painting of a squirrel eating a burger'
        lowerCAmelCase     =			torch.manual_seed(0    )
        lowerCAmelCase     =			sd_pipe(
            [prompt]    ,      generator=UpperCAmelCase__    ,      guidance_scale=7.5    ,      num_inference_steps=1_5    ,      output_type='np'    ,      use_karras_sigmas=UpperCAmelCase__    ,      )

        lowerCAmelCase     =			output.images

        lowerCAmelCase     =			image[0, -3:, -3:, -1]

        assert image.shape == (1, 5_1_2, 5_1_2, 3)
        lowerCAmelCase     =			np.array(
            [0.11_381_689, 0.12_112_921, 0.1_389_457, 0.12_549_606, 0.1_244_964, 0.10_831_517, 0.11_562_866, 0.10_867_816, 0.10_499_048]    )

        assert np.abs(image_slice.flatten() - expected_slice    ).max() < 1E-2









'''simple docstring'''




import multiprocessing
import time

from arguments import PretokenizationArguments
from datasets import load_dataset

from transformers import AutoTokenizer, HfArgumentParser





def 	a_					( lowerCamelCase	:     Dict				):
   lowerCAmelCase     =			{}
   lowerCAmelCase     =			tokenizer(example['content']	, truncation=lowerCamelCase				)['input_ids']
   lowerCAmelCase     =			len(example['content']				) / len(output['input_ids']				)
   return output


__snake_case    						=HfArgumentParser(PretokenizationArguments)
__snake_case    						=parser.parse_args()
if args.num_workers is None:
       __snake_case    						=multiprocessing.cpu_count()
__snake_case    						=AutoTokenizer.from_pretrained(args.tokenizer_dir)

__snake_case    						=time.time()
__snake_case    						=load_dataset(args.dataset_name, split="""train""")
print(F'''Dataset loaded in {time.time()-t_start:.2f}s''')

__snake_case    						=time.time()
__snake_case    						=ds.map(
    tokenize,
    num_proc=args.num_workers,
    remove_columns=[
        """repo_name""",
        """path""",
        """copies""",
        """size""",
        """content""",
        """license""",
        """hash""",
        """line_mean""",
        """line_max""",
        """alpha_frac""",
        """autogenerated""",
    ],
)
print(F'''Dataset tokenized in {time.time()-t_start:.2f}s''')

__snake_case    						=time.time()
ds.push_to_hub(args.tokenized_data_repo)




print(F'''Data pushed to the hub in {time.time()-t_start:.2f}s''')





