



'''simple docstring'''





from typing import Callable, List, Optional, Tuple, Union

import torch
from transformers import CLIPTextModel, CLIPTokenizer

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin, TransformeraDModel, VQModel
from ...schedulers import VQDiffusionScheduler
from ...utils import logging
from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput


_lowerCAmelCase      						=	logging.get_logger(__name__)  # pylint: disable=invalid-name






class        lowerCAmelCase_(				SCREAMING_SNAKE_CASE_						, SCREAMING_SNAKE_CASE_     ):
  '''simple docstring'''




  @register_to_config
  def __init__(    self  ,__UpperCAmelCase  ,__UpperCAmelCase = None  ,__UpperCAmelCase = None      )    ->		Dict:
      super().__init__()

      lowerCAmelCase__ :  List[Any]							      =					learnable

      if self.learnable:
          assert hidden_size is not None, "learnable=True requires `hidden_size` to be set"
          assert length is not None, "learnable=True requires `length` to be set"

          lowerCAmelCase__ :  Dict							      =					torch.zeros(__UpperCAmelCase  ,__UpperCAmelCase      )
      else:
          lowerCAmelCase__ :  Optional[int]							      =					None

      lowerCAmelCase__ :  Union[str, Any]							      =					torch.nn.Parameter(__UpperCAmelCase      )







class        lowerCAmelCase_(				SCREAMING_SNAKE_CASE_     ):
  '''simple docstring'''




  __lowercase							:      VQModel
  __lowercase							:      CLIPTextModel
  __lowercase							:      CLIPTokenizer
  __lowercase							:      TransformeraDModel
  __lowercase							:      LearnedClassifierFreeSamplingEmbeddings
  __lowercase							:      VQDiffusionScheduler




  def __init__(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,)    ->		Union[str, Any]:
      super().__init__()

      self.register_modules(
          vqvae=__UpperCAmelCase  ,transformer=__UpperCAmelCase  ,text_encoder=__UpperCAmelCase  ,tokenizer=__UpperCAmelCase  ,scheduler=__UpperCAmelCase  ,learned_classifier_free_sampling_embeddings=__UpperCAmelCase  ,)




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase      )    ->		Optional[int]:
      lowerCAmelCase__ :  Optional[int]							      =					len(__UpperCAmelCase      ) if isinstance(__UpperCAmelCase  ,__UpperCAmelCase      ) else 1

      # get prompt text embeddings
      lowerCAmelCase__ :  int							      =					self.tokenizer(
          __UpperCAmelCase  ,padding="""max_length"""  ,max_length=self.tokenizer.model_max_length  ,return_tensors="""pt"""  ,)
      lowerCAmelCase__ :  Dict							      =					text_inputs.input_ids

      if text_input_ids.shape[-1] > self.tokenizer.model_max_length:
          lowerCAmelCase__ :  List[Any]							      =					self.tokenizer.batch_decode(text_input_ids[:, self.tokenizer.model_max_length :]      )
          logger.warning(
              """The following part of your input was truncated because CLIP can only handle sequences up to"""
              F""" {self.tokenizer.model_max_length} tokens: {removed_text}"""      )
          lowerCAmelCase__ :  Optional[Any]							      =					text_input_ids[:, : self.tokenizer.model_max_length]
      lowerCAmelCase__ :  Union[str, Any]							      =					self.text_encoder(text_input_ids.to(self.device      )      )[0]

      # NOTE: This additional step of normalizing the text embeddings is from VQ-Diffusion.
      # While CLIP does normalize the pooled output of the text transformer when combining
      # the image and text embeddings, CLIP does not directly normalize the last hidden state.
      #
      # CLIP normalizing the pooled output.
      # https://github.com/huggingface/transformers/blob/d92e22d1f28324f513f3080e5c47c071a3916721/src/transformers/models/clip/modeling_clip.py#L1052-L1053
      lowerCAmelCase__ :  List[str]							      =					prompt_embeds / prompt_embeds.norm(dim=-1  ,keepdim=__UpperCAmelCase      )

      # duplicate text embeddings for each generation per prompt
      lowerCAmelCase__ :  Any							      =					prompt_embeds.repeat_interleave(__UpperCAmelCase  ,dim=0      )

      if do_classifier_free_guidance:
          if self.learned_classifier_free_sampling_embeddings.learnable:
              lowerCAmelCase__ :  Dict							      =					self.learned_classifier_free_sampling_embeddings.embeddings
              lowerCAmelCase__ :  Optional[Any]							      =					negative_prompt_embeds.unsqueeze(0      ).repeat(__UpperCAmelCase  ,1  ,1      )
          else:
              lowerCAmelCase__ :  Optional[Any]							      =					[""""""] * batch_size

              lowerCAmelCase__ :  int							      =					text_input_ids.shape[-1]
              lowerCAmelCase__ :  str							      =					self.tokenizer(
                  __UpperCAmelCase  ,padding="""max_length"""  ,max_length=__UpperCAmelCase  ,truncation=__UpperCAmelCase  ,return_tensors="""pt"""  ,)
              lowerCAmelCase__ :  Union[str, Any]							      =					self.text_encoder(uncond_input.input_ids.to(self.device      )      )[0]
              # See comment for normalizing text embeddings
              lowerCAmelCase__ :  Union[str, Any]							      =					negative_prompt_embeds / negative_prompt_embeds.norm(dim=-1  ,keepdim=__UpperCAmelCase      )

          # duplicate unconditional embeddings for each generation per prompt, using mps friendly method
          lowerCAmelCase__ :  Optional[int]							      =					negative_prompt_embeds.shape[1]
          lowerCAmelCase__ :  int							      =					negative_prompt_embeds.repeat(1  ,__UpperCAmelCase  ,1      )
          lowerCAmelCase__ :  Tuple							      =					negative_prompt_embeds.view(batch_size * num_images_per_prompt  ,__UpperCAmelCase  ,-1      )

          # For classifier free guidance, we need to do two forward passes.
          # Here we concatenate the unconditional and text embeddings into a single batch
          # to avoid doing two forward passes
          lowerCAmelCase__ :  List[Any]							      =					torch.cat([negative_prompt_embeds, prompt_embeds]      )

      return prompt_embeds




  @torch.no_grad()
  def __call__(    self  ,__UpperCAmelCase  ,__UpperCAmelCase = 100  ,__UpperCAmelCase = 5.0  ,__UpperCAmelCase = 1.0  ,__UpperCAmelCase = 1  ,__UpperCAmelCase = None  ,__UpperCAmelCase = None  ,__UpperCAmelCase = "pil"  ,__UpperCAmelCase = True  ,__UpperCAmelCase = None  ,__UpperCAmelCase = 1  ,)    ->		Union[ImagePipelineOutput, Tuple]:
      if isinstance(__UpperCAmelCase  ,__UpperCAmelCase      ):
          lowerCAmelCase__ :  Tuple							      =					1
      elif isinstance(__UpperCAmelCase  ,__UpperCAmelCase      ):
          lowerCAmelCase__ :  int							      =					len(__UpperCAmelCase      )
      else:
          raise ValueError(F"""`prompt` has to be of type `str` or `list` but is {type(__UpperCAmelCase      )}"""      )

      lowerCAmelCase__ :  Optional[Any]							      =					batch_size * num_images_per_prompt

      lowerCAmelCase__ :  Union[str, Any]							      =					guidance_scale > 1.0

      lowerCAmelCase__ :  Any							      =					self._encode_prompt(__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase      )

      if (callback_steps is None) or (
          callback_steps is not None and (not isinstance(__UpperCAmelCase  ,__UpperCAmelCase      ) or callback_steps <= 0)
      ):
          raise ValueError(
              F"""`callback_steps` has to be a positive integer but is {callback_steps} of type"""
              F""" {type(__UpperCAmelCase      )}."""      )

      # get the initial completely masked latents unless the user supplied it

      lowerCAmelCase__ :  Dict							      =					(batch_size, self.transformer.num_latent_pixels)
      if latents is None:
          lowerCAmelCase__ :  List[Any]							      =					self.transformer.num_vector_embeds - 1
          lowerCAmelCase__ :  int							      =					torch.full(__UpperCAmelCase  ,__UpperCAmelCase      ).to(self.device      )
      else:
          if latents.shape != latents_shape:
              raise ValueError(F"""Unexpected latents shape, got {latents.shape}, expected {latents_shape}"""      )
          if (latents < 0).any() or (latents >= self.transformer.num_vector_embeds).any():
              raise ValueError(
                  """Unexpected latents value(s). All latents be valid embedding indices i.e. in the range 0,"""
                  F""" {self.transformer.num_vector_embeds - 1} (inclusive)."""      )
          lowerCAmelCase__ :  List[str]							      =					latents.to(self.device      )

      # set timesteps
      self.scheduler.set_timesteps(__UpperCAmelCase  ,device=self.device      )

      lowerCAmelCase__ :  Any							      =					self.scheduler.timesteps.to(self.device      )

      lowerCAmelCase__ :  int							      =					latents

      for i, t in enumerate(self.progress_bar(__UpperCAmelCase      )      ):
          # expand the sample if we are doing classifier free guidance
          lowerCAmelCase__ :  Union[str, Any]							      =					torch.cat([sample] * 2      ) if do_classifier_free_guidance else sample

          # predict the un-noised image
          # model_output == `log_p_x_0`
          lowerCAmelCase__ :  List[str]							      =					self.transformer(__UpperCAmelCase  ,encoder_hidden_states=__UpperCAmelCase  ,timestep=__UpperCAmelCase      ).sample

          if do_classifier_free_guidance:
              lowerCAmelCase__      ,   lowerCAmelCase__ :  List[str]							      =					model_output.chunk(2      )
              lowerCAmelCase__ :  List[Any]							      =					model_output_uncond + guidance_scale * (model_output_text - model_output_uncond)
              model_output -= torch.logsumexp(__UpperCAmelCase  ,dim=1  ,keepdim=__UpperCAmelCase      )

          lowerCAmelCase__ :  List[str]							      =					self.truncate(__UpperCAmelCase  ,__UpperCAmelCase      )

          # remove `log(0)`'s (`-inf`s)
          lowerCAmelCase__ :  Optional[Any]							      =					model_output.clamp(-70      )

          # compute the previous noisy sample x_t -> x_t-1
          lowerCAmelCase__ :  List[str]							      =					self.scheduler.step(__UpperCAmelCase  ,timestep=__UpperCAmelCase  ,sample=__UpperCAmelCase  ,generator=__UpperCAmelCase      ).prev_sample

          # call the callback, if provided
          if callback is not None and i % callback_steps == 0:
              callback(__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase      )

      lowerCAmelCase__ :  Tuple							      =					self.vqvae.config.vq_embed_dim
      lowerCAmelCase__ :  Optional[int]							      =					(batch_size, self.transformer.height, self.transformer.width, embedding_channels)
      lowerCAmelCase__ :  List[str]							      =					self.vqvae.quantize.get_codebook_entry(__UpperCAmelCase  ,shape=__UpperCAmelCase      )
      lowerCAmelCase__ :  int							      =					self.vqvae.decode(__UpperCAmelCase  ,force_not_quantize=__UpperCAmelCase      ).sample

      lowerCAmelCase__ :  Dict							      =					(image / 2 + 0.5).clamp(0  ,1      )
      lowerCAmelCase__ :  Union[str, Any]							      =					image.cpu().permute(0  ,2  ,3  ,1      ).numpy()

      if output_type == "pil":
          lowerCAmelCase__ :  Dict							      =					self.numpy_to_pil(__UpperCAmelCase      )

      if not return_dict:
          return (image,)

      return ImagePipelineOutput(images=__UpperCAmelCase      )






  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase      )    ->		torch.FloatTensor:
      lowerCAmelCase__      ,   lowerCAmelCase__ :  Any							      =					torch.sort(__UpperCAmelCase  ,1  ,descending=__UpperCAmelCase      )
      lowerCAmelCase__ :  Any							      =					torch.exp(__UpperCAmelCase      )
      lowerCAmelCase__ :  Dict							      =					sorted_p_x_0.cumsum(dim=1      ) < truncation_rate

      # Ensure that at least the largest probability is not zeroed out
      lowerCAmelCase__ :  Dict							      =					torch.full_like(keep_mask[:, 0:1, :]  ,__UpperCAmelCase      )
      lowerCAmelCase__ :  Optional[int]							      =					torch.cat((all_true, keep_mask)  ,dim=1      )
      lowerCAmelCase__ :  str							      =					keep_mask[:, :-1, :]

      lowerCAmelCase__ :  Optional[Any]							      =					keep_mask.gather(1  ,indices.argsort(1      )      )

      lowerCAmelCase__ :  List[Any]							      =					log_p_x_0.clone()

      lowerCAmelCase__ :  Optional[Any]							      =					-torch.inf  # -inf = log(0)

      return rv





"""simple docstring"""





import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
   from sagemaker import Session, TrainingJobAnalytics
   from sagemaker.huggingface import HuggingFace




@pytest.mark.skipif(
    literal_eval(os.getenv("TEST_SAGEMAKER"     ,"False"   )   ) is not True     ,reason="Skipping test because should only be run when releasing minor transformers version"     ,)
@pytest.mark.usefixtures("sm_env"   )
@parameterized_class(
    [
        {
            "framework": "pytorch",
            "script": "run_glue_model_parallelism.py",
            "model_name_or_path": "roberta-large",
            "instance_type": "ml.p3dn.24xlarge",
            "results": {"train_runtime": 1_6_0_0, "eval_accuracy": 0.3, "eval_loss": 1.2},
        },
        {
            "framework": "pytorch",
            "script": "run_glue.py",
            "model_name_or_path": "roberta-large",
            "instance_type": "ml.p3dn.24xlarge",
            "results": {"train_runtime": 1_6_0_0, "eval_accuracy": 0.3, "eval_loss": 1.2},
        },
    ]   )
class 					_lowerCAmelCase  (	unittest.TestCase   ):







 """simple docstring"""







 def 					_lowercase (			self							:			Optional[int]	):
    if self.framework == "pytorch":
       subprocess.run(
           F"""cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py""".split(),					encoding="utf-8",					check=UpperCAmelCase__,					)
    assert hasattr(self,					"env"	)


 def 					_lowercase (			self							:			str,					UpperCAmelCase__							:			List[Any]	):
    # configuration for running training on smdistributed Model Parallel
    __lowercase								=						{
        "enabled": True,
        "processes_per_host": 8,
    }
    __lowercase								=						{
        "enabled": True,
        "parameters": {
            "microbatches": 4,
            "placement_strategy": "spread",
            "pipeline": "interleaved",
            "optimize": "speed",
            "partitions": 4,
            "ddp": True,
        },
    }

    __lowercase								=						{"smdistributed": {"modelparallel": smp_options}, "mpi": mpi_options}

    __lowercase								=						"trainer" if self.script == "run_glue.py" else "smtrainer"
    # creates estimator
    return HuggingFace(
        entry_point=self.script,					source_dir=self.env.test_path,					role=self.env.role,					image_uri=self.env.image_uri,					base_job_name=F"""{self.env.base_job_name}-{instance_count}-smp-{name_extension}""",					instance_count=UpperCAmelCase__,					instance_type=self.instance_type,					debugger_hook_config=UpperCAmelCase__,					hyperparameters={
            **self.env.hyperparameters,
            "model_name_or_path": self.model_name_or_path,
            "max_steps": 5_0_0,
        },					metric_definitions=self.env.metric_definitions,					distribution=UpperCAmelCase__,					py_version="py36",					)


 def 					_lowercase (			self							:			Tuple,					UpperCAmelCase__							:			int	):
    TrainingJobAnalytics(UpperCAmelCase__	).export_csv(F"""{self.env.test_path}/{job_name}_metrics.csv"""	)



 @parameterized.expand([(1,)]	)
 def 					_lowercase (			self							:			str,					UpperCAmelCase__							:			Union[str, Any]	):
    # create estimator
    __lowercase								=						self.create_estimator(UpperCAmelCase__	)

    # run training
    estimator.fit()

    # result dataframe
    __lowercase								=						TrainingJobAnalytics(estimator.latest_training_job.name	).dataframe()

    # extract kpis
    __lowercase								=						list(result_metrics_df[result_metrics_df.metric_name == "eval_accuracy"]["value"]	)
    __lowercase								=						list(result_metrics_df[result_metrics_df.metric_name == "eval_loss"]["value"]	)
    # get train time from SageMaker job, this includes starting, preprocessing, stopping
    __lowercase								=						(
        Session().describe_training_job(estimator.latest_training_job.name	).get("TrainingTimeInSeconds",					9_9_9_9_9_9	)
    )

    # assert kpis
    assert train_runtime <= self.results["train_runtime"]
    assert all(t >= self.results["eval_accuracy"] for t in eval_accuracy	)
    assert all(t <= self.results["eval_loss"] for t in eval_loss	)

    # dump tests result into json file to share in PR
    with open(F"""{estimator.latest_training_job.name}.json""",					"w"	) as outfile:
       json.dump({"train_time": train_runtime, "eval_accuracy": eval_accuracy, "eval_loss": eval_loss},					UpperCAmelCase__	)




