



import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader


class    _lowercase (a_   ):
				'''simple docstring'''







				def __init__(			self  ,			snake_case__  ,			snake_case__ = None  ,			snake_case__ = None  ,			snake_case__ = None  ,			snake_case__ = False  ,			snake_case__ = False  ,			snake_case__ = None  ,			**snake_case__  ,			):

									'''simple docstring'''


									super().__init__(
									    snake_case__  ,			split=snake_case__  ,			features=snake_case__  ,			cache_dir=snake_case__  ,			keep_in_memory=snake_case__  ,			streaming=snake_case__  ,			num_proc=snake_case__  ,			**snake_case__  ,			)
									UpperCamelCase_			= path_or_paths if isinstance(snake_case__  ,			snake_case__     ) else {self.split: path_or_paths}
									UpperCamelCase_			= Csv(
									    cache_dir=snake_case__  ,			data_files=snake_case__  ,			features=snake_case__  ,			**snake_case__  ,			)

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									if self.streaming:
														UpperCamelCase_			= self.builder.as_streaming_dataset(split=self.split     )
									# Build regular (map-style) dataset
									else:
														UpperCamelCase_			= None
														UpperCamelCase_			= None
														UpperCamelCase_			= None
														UpperCamelCase_			= None

														self.builder.download_and_prepare(
														    download_config=snake_case__  ,			download_mode=snake_case__  ,			verification_mode=snake_case__  ,			base_path=snake_case__  ,			num_proc=self.num_proc  ,			)
														UpperCamelCase_			= self.builder.as_dataset(
														    split=self.split  ,			verification_mode=snake_case__  ,			in_memory=self.keep_in_memory     )
									return dataset




class    _lowercase :
				'''simple docstring'''







				def __init__(			self  ,			snake_case__  ,			snake_case__  ,			snake_case__ = None  ,			snake_case__ = None  ,			**snake_case__  ,			):

									'''simple docstring'''


									if num_proc is not None and num_proc <= 0:
														raise ValueError(F"""num_proc {num_proc} must be an integer > 0."""     )

									UpperCamelCase_			= dataset
									UpperCamelCase_			= path_or_buf
									UpperCamelCase_			= batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
									UpperCamelCase_			= num_proc
									UpperCamelCase_			= "utf-8"
									UpperCamelCase_			= to_csv_kwargs

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= self.to_csv_kwargs.pop("path_or_buf"  ,			snake_case__     )
									UpperCamelCase_			= self.to_csv_kwargs.pop("header"  ,			snake_case__     )
									UpperCamelCase_			= self.to_csv_kwargs.pop("index"  ,			snake_case__     )

									if isinstance(self.path_or_buf  ,			(str, bytes, os.PathLike)     ):
														with open(self.path_or_buf  ,			"wb+"     ) as buffer:
																			UpperCamelCase_			= self._write(file_obj=snake_case__  ,			header=snake_case__  ,			index=snake_case__  ,			**self.to_csv_kwargs     )
									else:
														UpperCamelCase_			= self._write(file_obj=self.path_or_buf  ,			header=snake_case__  ,			index=snake_case__  ,			**self.to_csv_kwargs     )
									return written

				def    _lowerCamelCase       (			self  ,			snake_case__     ):

									'''simple docstring'''


									UpperCamelCase_	,     UpperCamelCase_	,     UpperCamelCase_	,     UpperCamelCase_			= args

									UpperCamelCase_			= query_table(
									    table=self.dataset.data  ,			key=slice(snake_case__  ,			offset + self.batch_size     )  ,			indices=self.dataset._indices  ,			)
									UpperCamelCase_			= batch.to_pandas().to_csv(
									    path_or_buf=snake_case__  ,			header=header if (offset == 0) else False  ,			index=snake_case__  ,			**snake_case__     )
									return csv_str.encode(self.encoding     )

				def    _lowerCamelCase       (			self  ,			snake_case__  ,			snake_case__  ,			snake_case__  ,			**snake_case__     ):

									'''simple docstring'''


									UpperCamelCase_			= 0

									if self.num_proc is None or self.num_proc == 1:
														for offset in logging.tqdm(
														    range(0  ,			len(self.dataset     )  ,			self.batch_size     )  ,			unit="ba"  ,			disable=not logging.is_progress_bar_enabled()  ,			desc="Creating CSV from Arrow format"  ,			):
																			UpperCamelCase_			= self._batch_csv((offset, header, index, to_csv_kwargs)     )
																			written += file_obj.write(snake_case__     )

									else:
														UpperCamelCase_	,     UpperCamelCase_			= len(self.dataset     ), self.batch_size
														with multiprocessing.Pool(self.num_proc     ) as pool:
																			for csv_str in logging.tqdm(
																			    pool.imap(
																			        self._batch_csv  ,			[(offset, header, index, to_csv_kwargs) for offset in range(0  ,			snake_case__  ,			snake_case__     )]  ,			)  ,			total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size  ,			unit="ba"  ,			disable=not logging.is_progress_bar_enabled()  ,			desc="Creating CSV from Arrow format"  ,			):
																								written += file_obj.write(snake_case__     )

									return written





'''simple docstring'''







import numpy as np
import torch
import torch.nn as nn
from transformers import CLIPConfig, CLIPVisionModel, PreTrainedModel

from ...utils import logging


SCREAMING_SNAKE_CASE__     : Optional[int]				=   logging.get_logger(__name__)







def  SCREAMING_SNAKE_CASE__ (				SCREAMING_SNAKE_CASE ,		SCREAMING_SNAKE_CASE     ):
		SCREAMING_SNAKE_CASE_						:List[Any]										=      nn.functional.normalize(SCREAMING_SNAKE_CASE     )
		SCREAMING_SNAKE_CASE_						:str										=      nn.functional.normalize(SCREAMING_SNAKE_CASE     )
		return torch.mm(SCREAMING_SNAKE_CASE ,		normalized_text_embeds.t()     )





class 				__lowerCAmelCase(							lowerCAmelCase__      ):
			__snake_case  :		int		=      CLIPConfig

			__snake_case  :		Optional[Any]		=      ['CLIPEncoderLayer']






			def __init__(   self  :							int      ,       SCREAMING_SNAKE_CASE  :							CLIPConfig	):




					"""simple docstring"""







					super().__init__(SCREAMING_SNAKE_CASE	)

					SCREAMING_SNAKE_CASE_						:List[Any]										=      CLIPVisionModel(config.vision_config	)
					SCREAMING_SNAKE_CASE_						:Dict										=      nn.Linear(config.vision_config.hidden_size      ,       config.projection_dim      ,       bias=SCREAMING_SNAKE_CASE	)

					SCREAMING_SNAKE_CASE_						:List[Any]										=      nn.Parameter(torch.ones(17      ,       config.projection_dim	)      ,       requires_grad=SCREAMING_SNAKE_CASE	)
					SCREAMING_SNAKE_CASE_						:Optional[int]										=      nn.Parameter(torch.ones(3      ,       config.projection_dim	)      ,       requires_grad=SCREAMING_SNAKE_CASE	)

					SCREAMING_SNAKE_CASE_						:Union[str, Any]										=      nn.Parameter(torch.ones(17	)      ,       requires_grad=SCREAMING_SNAKE_CASE	)
					SCREAMING_SNAKE_CASE_						:str										=      nn.Parameter(torch.ones(3	)      ,       requires_grad=SCREAMING_SNAKE_CASE	)






			@torch.no_grad()
			def 				_lowercase	(   self  :							Tuple      ,       SCREAMING_SNAKE_CASE  :							Tuple      ,       SCREAMING_SNAKE_CASE  :							Tuple	):




					"""simple docstring"""







					SCREAMING_SNAKE_CASE_						:Optional[int]										=      self.vision_model(SCREAMING_SNAKE_CASE	)[1]  # pooled_output
					SCREAMING_SNAKE_CASE_						:Dict										=      self.visual_projection(SCREAMING_SNAKE_CASE	)

					# we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16
					SCREAMING_SNAKE_CASE_						:Union[str, Any]										=      cosine_distance(SCREAMING_SNAKE_CASE      ,       self.special_care_embeds	).cpu().float().numpy()
					SCREAMING_SNAKE_CASE_						:List[Any]										=      cosine_distance(SCREAMING_SNAKE_CASE      ,       self.concept_embeds	).cpu().float().numpy()

					SCREAMING_SNAKE_CASE_						:str										=      []
					SCREAMING_SNAKE_CASE_						:Optional[Any]										=      image_embeds.shape[0]
					for i in range(SCREAMING_SNAKE_CASE	):
							SCREAMING_SNAKE_CASE_						:List[Any]										=      {'special_scores': {}, 'special_care': [], 'concept_scores': {}, 'bad_concepts': []}

							# increase this value to create a stronger `nfsw` filter
							# at the cost of increasing the possibility of filtering benign images
							SCREAMING_SNAKE_CASE_						:Optional[int]										=      0.0

							for concept_idx in range(len(special_cos_dist[0]	)	):
									SCREAMING_SNAKE_CASE_						:Dict										=      special_cos_dist[i][concept_idx]
									SCREAMING_SNAKE_CASE_						:Tuple										=      self.special_care_embeds_weights[concept_idx].item()
									SCREAMING_SNAKE_CASE_						:Any										=      round(concept_cos - concept_threshold + adjustment      ,       3	)
									if result_img["special_scores"][concept_idx] > 0:
											result_img["special_care"].append({concept_idx, result_img['special_scores'][concept_idx]}	)
											SCREAMING_SNAKE_CASE_						:Union[str, Any]										=      0.01

							for concept_idx in range(len(cos_dist[0]	)	):
									SCREAMING_SNAKE_CASE_						:int										=      cos_dist[i][concept_idx]
									SCREAMING_SNAKE_CASE_						:Tuple										=      self.concept_embeds_weights[concept_idx].item()
									SCREAMING_SNAKE_CASE_						:Union[str, Any]										=      round(concept_cos - concept_threshold + adjustment      ,       3	)
									if result_img["concept_scores"][concept_idx] > 0:
											result_img["bad_concepts"].append(SCREAMING_SNAKE_CASE	)

							result.append(SCREAMING_SNAKE_CASE	)

					SCREAMING_SNAKE_CASE_						:Any										=      [len(res['bad_concepts']	) > 0 for res in result]

					for idx, has_nsfw_concept in enumerate(SCREAMING_SNAKE_CASE	):
							if has_nsfw_concept:
									if torch.is_tensor(SCREAMING_SNAKE_CASE	) or torch.is_tensor(images[0]	):
											SCREAMING_SNAKE_CASE_						:Tuple										=      torch.zeros_like(images[idx]	)  # black image
									else:
											SCREAMING_SNAKE_CASE_						:str										=      np.zeros(images[idx].shape	)  # black image

					if any(SCREAMING_SNAKE_CASE	):
							logger.warning(
							    'Potential NSFW content was detected in one or more images. A black image will be returned instead.'
							    ' Try again with a different prompt and/or seed.'	)

					return images, has_nsfw_concepts






			@torch.no_grad()
			def 				_lowercase	(   self  :							str      ,       SCREAMING_SNAKE_CASE  :							torch.FloatTensor      ,       SCREAMING_SNAKE_CASE  :							torch.FloatTensor	):




					"""simple docstring"""







					SCREAMING_SNAKE_CASE_						:Tuple										=      self.vision_model(SCREAMING_SNAKE_CASE	)[1]  # pooled_output
					SCREAMING_SNAKE_CASE_						:Tuple										=      self.visual_projection(SCREAMING_SNAKE_CASE	)

					SCREAMING_SNAKE_CASE_						:List[str]										=      cosine_distance(SCREAMING_SNAKE_CASE      ,       self.special_care_embeds	)
					SCREAMING_SNAKE_CASE_						:int										=      cosine_distance(SCREAMING_SNAKE_CASE      ,       self.concept_embeds	)

					# increase this value to create a stronger `nsfw` filter
					# at the cost of increasing the possibility of filtering benign images
					SCREAMING_SNAKE_CASE_						:Dict										=      0.0

					SCREAMING_SNAKE_CASE_						:List[str]										=      special_cos_dist - self.special_care_embeds_weights + adjustment
					# special_scores = special_scores.round(decimals=3)
					SCREAMING_SNAKE_CASE_						:Dict										=      torch.any(special_scores > 0      ,       dim=1	)
					SCREAMING_SNAKE_CASE_						:Dict										=      special_care * 0.01
					SCREAMING_SNAKE_CASE_						:Union[str, Any]										=      special_adjustment.unsqueeze(1	).expand(-1      ,       cos_dist.shape[1]	)

					SCREAMING_SNAKE_CASE_						:str										=      (cos_dist - self.concept_embeds_weights) + special_adjustment
					# concept_scores = concept_scores.round(decimals=3)
					SCREAMING_SNAKE_CASE_						:str										=      torch.any(concept_scores > 0      ,       dim=1	)

					SCREAMING_SNAKE_CASE_						:Optional[int]										=      0.0  # black image

					return images, has_nsfw_concepts
