
'''simple docstring'''





import importlib
import os
import platform
from pathlib import Path

from packaging import version

from .utils.logging import get_logger


UpperCAmelCase            = get_logger(__name__)

# Datasets
UpperCAmelCase            = "https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets"
UpperCAmelCase            = "https://cdn-datasets.huggingface.co/datasets/datasets"
UpperCAmelCase            = "https://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}"

# Metrics
UpperCAmelCase            = "https://s3.amazonaws.com/datasets.huggingface.co/datasets/metrics"
UpperCAmelCase            = "https://cdn-datasets.huggingface.co/datasets/metric"
UpperCAmelCase            = "https://raw.githubusercontent.com/huggingface/datasets/{revision}/metrics/{path}/{name}"

# Hub
UpperCAmelCase            = os.environ.get("HF_ENDPOINT", "https://huggingface.co")
UpperCAmelCase            = HF_ENDPOINT + "/datasets/{repo_id}/resolve/{revision}/{path}"
UpperCAmelCase            = "main"

UpperCAmelCase            = version.parse(platform.python_version())

if PY_VERSION < version.parse("3.8"):
						import importlib_metadata
else:
						import importlib.metadata as importlib_metadata

# General environment variables accepted values for booleans
UpperCAmelCase            = {"1", "ON", "YES", "TRUE"}
UpperCAmelCase            = ENV_VARS_TRUE_VALUES.union({"AUTO"})


# Imports
UpperCAmelCase            = version.parse(importlib_metadata.version("dill"))
UpperCAmelCase            = version.parse(importlib_metadata.version("pandas"))
UpperCAmelCase            = version.parse(importlib_metadata.version("pyarrow"))

UpperCAmelCase            = os.environ.get("USE_TF", "AUTO").upper()
UpperCAmelCase            = os.environ.get("USE_TORCH", "AUTO").upper()
UpperCAmelCase            = os.environ.get("USE_JAX", "AUTO").upper()

UpperCAmelCase            = "N/A"
UpperCAmelCase            = False

if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES:
						UpperCAmelCase            = importlib.util.find_spec("torch") is not None
						if TORCH_AVAILABLE:
												try:
																		UpperCAmelCase            = version.parse(importlib_metadata.version("torch"))
																		logger.info(f'''PyTorch version {TORCH_VERSION} available.''')
												except importlib_metadata.PackageNotFoundError:
																		pass
else:
						logger.info("Disabling PyTorch because USE_TF is set")

UpperCAmelCase            = "N/A"
UpperCAmelCase            = False

if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES:
						UpperCAmelCase            = importlib.util.find_spec("tensorflow") is not None
						if TF_AVAILABLE:
												# For the metadata, we have to look for both tensorflow and tensorflow-cpu
												for package in [
												    "tensorflow",
												    "tensorflow-cpu",
												    "tensorflow-gpu",
												    "tf-nightly",
												    "tf-nightly-cpu",
												    "tf-nightly-gpu",
												    "intel-tensorflow",
												    "tensorflow-rocm",
												    "tensorflow-macos",
												]:
																		try:
																								UpperCAmelCase            = version.parse(importlib_metadata.version(package))
																		except importlib_metadata.PackageNotFoundError:
																								continue
																		else:
																								break
												else:
																		UpperCAmelCase            = False
						if TF_AVAILABLE:
												if TF_VERSION.major < 2:
																		logger.info(f'''TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.''')
																		UpperCAmelCase            = False
												else:
																		logger.info(f'''TensorFlow version {TF_VERSION} available.''')
else:
						logger.info("Disabling Tensorflow because USE_TORCH is set")


UpperCAmelCase            = "N/A"
UpperCAmelCase            = False

if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES:
						UpperCAmelCase            = importlib.util.find_spec("jax") is not None and importlib.util.find_spec("jaxlib") is not None
						if JAX_AVAILABLE:
												try:
																		UpperCAmelCase            = version.parse(importlib_metadata.version("jax"))
																		logger.info(f'''JAX version {JAX_VERSION} available.''')
												except importlib_metadata.PackageNotFoundError:
																		pass
else:
						logger.info("Disabling JAX because USE_JAX is set to False")


UpperCAmelCase            = os.environ.get("USE_BEAM", "AUTO").upper()
UpperCAmelCase            = "N/A"
UpperCAmelCase            = False
if USE_BEAM in ENV_VARS_TRUE_AND_AUTO_VALUES:
						try:
												UpperCAmelCase            = version.parse(importlib_metadata.version("apache_beam"))
												UpperCAmelCase            = True
												logger.info(f'''Apache Beam version {BEAM_VERSION} available.''')
						except importlib_metadata.PackageNotFoundError:
												pass
else:
						logger.info("Disabling Apache Beam because USE_BEAM is set to False")


# Optional tools for data loading
UpperCAmelCase            = importlib.util.find_spec("sqlalchemy") is not None

# Optional tools for feature decoding
UpperCAmelCase            = importlib.util.find_spec("PIL") is not None
UpperCAmelCase            = importlib.util.find_spec("soundfile") is not None and version.parse(
    importlib.import_module("soundfile").__libsndfile_version__
) >= version.parse("1.0.31")
UpperCAmelCase            = importlib.util.find_spec("soundfile") is not None and version.parse(
    importlib.import_module("soundfile").__libsndfile_version__
) >= version.parse("1.1.0")

# Optional compression tools
UpperCAmelCase            = importlib.util.find_spec("rarfile") is not None
UpperCAmelCase            = importlib.util.find_spec("zstandard") is not None
UpperCAmelCase            = importlib.util.find_spec("lz4") is not None
UpperCAmelCase            = importlib.util.find_spec("py7zr") is not None


# Cache location
UpperCAmelCase            = "~/.cache"
UpperCAmelCase            = os.getenv("XDG_CACHE_HOME", DEFAULT_XDG_CACHE_HOME)
UpperCAmelCase            = os.path.join(XDG_CACHE_HOME, "huggingface")
UpperCAmelCase            = os.path.expanduser(os.getenv("HF_HOME", DEFAULT_HF_CACHE_HOME))

UpperCAmelCase            = os.path.join(HF_CACHE_HOME, "datasets")
UpperCAmelCase            = Path(os.getenv("HF_DATASETS_CACHE", DEFAULT_HF_DATASETS_CACHE))

UpperCAmelCase            = os.path.join(HF_CACHE_HOME, "metrics")
UpperCAmelCase            = Path(os.getenv("HF_METRICS_CACHE", DEFAULT_HF_METRICS_CACHE))

UpperCAmelCase            = os.path.join(HF_CACHE_HOME, "modules")
UpperCAmelCase            = Path(os.getenv("HF_MODULES_CACHE", DEFAULT_HF_MODULES_CACHE))

UpperCAmelCase            = "downloads"
UpperCAmelCase            = os.path.join(HF_DATASETS_CACHE, DOWNLOADED_DATASETS_DIR)
UpperCAmelCase            = Path(os.getenv("HF_DATASETS_DOWNLOADED_DATASETS_PATH", DEFAULT_DOWNLOADED_DATASETS_PATH))

UpperCAmelCase            = "extracted"
UpperCAmelCase            = os.path.join(DEFAULT_DOWNLOADED_DATASETS_PATH, EXTRACTED_DATASETS_DIR)
UpperCAmelCase            = Path(os.getenv("HF_DATASETS_EXTRACTED_DATASETS_PATH", DEFAULT_EXTRACTED_DATASETS_PATH))

# Download count for the website
UpperCAmelCase            = (
    os.environ.get("HF_UPDATE_DOWNLOAD_COUNTS", "AUTO").upper() in ENV_VARS_TRUE_AND_AUTO_VALUES
)

# Batch size constants. For more info, see:
# https://github.com/apache/arrow/blob/master/docs/source/cpp/arrays.rst#size-limitations-and-recommendations)
UpperCAmelCase            = 1_000

# Size of the preloaded record batch in `Dataset.__iter__`
UpperCAmelCase            = 10

# Pickling tables works only for small tables (<4GiB)
# For big tables, we write them on disk instead
UpperCAmelCase            = 4 << 30

# Max shard size in bytes (e.g. to shard parquet datasets in push_to_hub or download_and_prepare)
UpperCAmelCase            = "500MB"

# Parquet configuration
UpperCAmelCase            = 100
UpperCAmelCase            = 100
UpperCAmelCase            = 100

# Offline mode
UpperCAmelCase            = os.environ.get("HF_DATASETS_OFFLINE", "AUTO").upper() in ENV_VARS_TRUE_VALUES

# In-memory
UpperCAmelCase            = 0  # Disabled
UpperCAmelCase            = float(os.environ.get("HF_DATASETS_IN_MEMORY_MAX_SIZE", DEFAULT_IN_MEMORY_MAX_SIZE))

# File names
UpperCAmelCase            = "dataset.arrow"
UpperCAmelCase            = "indices.arrow"
UpperCAmelCase            = "state.json"
UpperCAmelCase            = "dataset_info.json"
UpperCAmelCase            = "dataset_infos.json"
UpperCAmelCase            = "LICENSE"
UpperCAmelCase            = "metric_info.json"
UpperCAmelCase            = "dataset_dict.json"

UpperCAmelCase            = "datasets_modules"

UpperCAmelCase            = 255

# Streaming
UpperCAmelCase            = 20
UpperCAmelCase            = 5

# Datasets without script
UpperCAmelCase            = 200
UpperCAmelCase            = 10
UpperCAmelCase            = 200

# Progress bars
UpperCAmelCase            = 0.05  # 20 progress updates per sec



'''simple docstring'''





import os
import pathlib
import tempfile
import uuid

import numpy as np

from ..utils import is_soundfile_availble, is_torch_available, is_vision_available, logging


UpperCAmelCase            = logging.get_logger(__name__)

if is_vision_available():
						import PIL.Image
						from PIL import Image
						from PIL.Image import Image as ImageType
else:
						UpperCAmelCase            = object

if is_torch_available():
						import torch

if is_soundfile_availble():
						import soundfile as sf

class     a :




				def __init__(				self						:					Dict,     SCREAMING_SNAKE_CASE_						:					Union[str, Any]   ):
										snake_case							:  Any			=     value




				def __str__(				self						:					Optional[int]   ):
										return self.to_string()




				def 						__snake_case				(				self						:					Optional[Any]   ):
										logger.error(
										    '''This is a raw AgentType of unknown type. Display in notebooks and string conversion will be unreliable'''   )
										return self._value




				def 						__snake_case				(				self						:					List[Any]   ):
										logger.error(
										    '''This is a raw AgentType of unknown type. Display in notebooks and string conversion will be unreliable'''   )
										return str(self._value   )




class     a (					__magic_name__					,__magic_name__     ):




				def 						__snake_case				(				self						:					Optional[Any]   ):
										return self._value




				def 						__snake_case				(				self						:					str   ):
										return self._value




class     a (					__magic_name__					,__magic_name__     ):




				def __init__(				self						:					Dict,     SCREAMING_SNAKE_CASE_						:					str   ):
										super().__init__(SCREAMING_SNAKE_CASE_   )

										if not is_vision_available():
																raise ImportError('''PIL must be installed in order to handle images.'''   )

										snake_case							:  Union[str, Any]			=     None
										snake_case							:  List[Any]			=     None
										snake_case							:  List[str]			=     None

										if isinstance(SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_   ):
																snake_case							:  List[str]			=     value
										elif isinstance(SCREAMING_SNAKE_CASE_,     (str, pathlib.Path)   ):
																snake_case							:  Any			=     value
										elif isinstance(SCREAMING_SNAKE_CASE_,     torch.Tensor   ):
																snake_case							:  Union[str, Any]			=     value
										else:
																raise ValueError(F"""Unsupported type for {self.__class__.__name__}: {type(SCREAMING_SNAKE_CASE_   )}"""   )




				def 						__snake_case				(				self						:					Tuple,     SCREAMING_SNAKE_CASE_						:					Any=None,     SCREAMING_SNAKE_CASE_						:					Any=None   ):
										from IPython.display import Image, display

										display(Image(self.to_string()   )   )




				def 						__snake_case				(				self						:					List[Any]   ):
										if self._raw is not None:
																return self._raw

										if self._path is not None:
																snake_case							:  str			=     Image.open(self._path   )
																return self._raw




				def 						__snake_case				(				self						:					Union[str, Any]   ):
										if self._path is not None:
																return self._path

										if self._raw is not None:
																snake_case							:  int			=     tempfile.mkdtemp()
																snake_case							:  str			=     os.path.join(SCREAMING_SNAKE_CASE_,     str(uuid.uuida()   ) + '''.png'''   )
																self._raw.save(self._path   )

																return self._path

										if self._tensor is not None:
																snake_case							:  Tuple			=     self._tensor.cpu().detach().numpy()

																# There is likely simpler than load into image into save
																snake_case							:  str			=     Image.fromarray((array * 2_55).astype(np.uinta   )   )

																snake_case							:  List[Any]			=     tempfile.mkdtemp()
																snake_case							:  Optional[Any]			=     os.path.join(SCREAMING_SNAKE_CASE_,     str(uuid.uuida()   ) + '''.png'''   )

																img.save(self._path   )

																return self._path




class     a (					__magic_name__     ):




				def __init__(				self						:					List[Any],     SCREAMING_SNAKE_CASE_						:					Tuple,     SCREAMING_SNAKE_CASE_						:					Optional[int]=1_60_00   ):
										super().__init__(SCREAMING_SNAKE_CASE_   )

										if not is_soundfile_availble():
																raise ImportError('''soundfile must be installed in order to handle audio.'''   )

										snake_case							:  List[Any]			=     None
										snake_case							:  int			=     None

										snake_case							:  Tuple			=     samplerate

										if isinstance(SCREAMING_SNAKE_CASE_,     (str, pathlib.Path)   ):
																snake_case							:  int			=     value
										elif isinstance(SCREAMING_SNAKE_CASE_,     torch.Tensor   ):
																snake_case							:  List[Any]			=     value
										else:
																raise ValueError(F"""Unsupported audio type: {type(SCREAMING_SNAKE_CASE_   )}"""   )




				def 						__snake_case				(				self						:					str,     SCREAMING_SNAKE_CASE_						:					str=None,     SCREAMING_SNAKE_CASE_						:					Optional[int]=None   ):
										from IPython.display import Audio, display

										display(Audio(self.to_string(),     rate=self.samplerate   )   )




				def 						__snake_case				(				self						:					List[str]   ):
										if self._tensor is not None:
																return self._tensor

										if self._path is not None:
																snake_case,       snake_case							:  Optional[Any]			=     sf.read(self._path   )
																snake_case							:  Any			=     torch.tensor(SCREAMING_SNAKE_CASE_   )
																return self._tensor




				def 						__snake_case				(				self						:					List[Any]   ):
										if self._path is not None:
																return self._path

										if self._tensor is not None:
																snake_case							:  Optional[Any]			=     tempfile.mkdtemp()
																snake_case							:  List[str]			=     os.path.join(SCREAMING_SNAKE_CASE_,     str(uuid.uuida()   ) + '''.wav'''   )
																sf.write(self._path,     self._tensor,     samplerate=self.samplerate   )
																return self._path




UpperCAmelCase            = {"text": AgentText, "image": AgentImage, "audio": AgentAudio}
UpperCAmelCase            = {str: AgentText}

if is_vision_available():
						UpperCAmelCase            = AgentImage

def   A			(							*A_				:       List[str]     ,				**A_				:       Any							):
						snake_case							:  str			=     [(arg.to_raw() if isinstance(A_     ,				A_							) else arg) for arg in args]
						snake_case							:  Dict			=     {k: (v.to_raw() if isinstance(A_     ,				A_							) else v) for k, v in kwargs.items()}
						return args, kwargs





def   A			(							A_				:       Union[str, Any]     ,				A_				:       Any=None							):
						if isinstance(A_     ,				A_							):
												snake_case							:  Union[str, Any]			=     {}
												for i, (k, v) in enumerate(outputs.items()							):
																		if output_types is not None:
																								# If the class has defined outputs, we can map directly according to the class definition
																								if output_types[i] in AGENT_TYPE_MAPPING:
																														snake_case							:  Optional[int]			=     AGENT_TYPE_MAPPING[output_types[i]](A_							)
																								else:
																														snake_case							:  List[str]			=     AgentType(A_							)

																		else:
																								# If the class does not have defined output, then we map according to the type
																								for _k, _v in INSTANCE_TYPE_MAPPING.items():
																														if isinstance(A_     ,				_k							):
																																				snake_case							:  Dict			=     _v(A_							)
																								if k not in decoded_outputs:
																														snake_case							:  List[str]			=     AgentType[v]

						elif isinstance(A_     ,				(list, tuple)							):
												snake_case							:  Union[str, Any]			=     type(A_							)()
												for i, v in enumerate(A_							):
																		if output_types is not None:
																								# If the class has defined outputs, we can map directly according to the class definition
																								if output_types[i] in AGENT_TYPE_MAPPING:
																														decoded_outputs.append(AGENT_TYPE_MAPPING[output_types[i]](A_							)							)
																								else:
																														decoded_outputs.append(AgentType(A_							)							)
																		else:
																								# If the class does not have defined output, then we map according to the type
																								snake_case							:  Union[str, Any]			=     False
																								for _k, _v in INSTANCE_TYPE_MAPPING.items():
																														if isinstance(A_     ,				_k							):
																																				decoded_outputs.append(_v(A_							)							)
																																				snake_case							:  Tuple			=     True

																								if not found:
																														decoded_outputs.append(AgentType(A_							)							)

						else:
												if output_types[0] in AGENT_TYPE_MAPPING:
																		# If the class has defined outputs, we can map directly according to the class definition
																		snake_case							:  List[str]			=     AGENT_TYPE_MAPPING[output_types[0]](A_							)

												else:
																		# If the class does not have defined output, then we map according to the type
																		for _k, _v in INSTANCE_TYPE_MAPPING.items():
																								if isinstance(A_     ,				_k							):
																														return _v(A_							)
																		return AgentType(A_							)

						return decoded_outputs

