




import importlib
import os
import platform
from pathlib import Path

from packaging import version

from .utils.logging import get_logger


UpperCamelCase__						=		get_logger(__name__)

# Datasets
UpperCamelCase__						=		'https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets'
UpperCamelCase__						=		'https://cdn-datasets.huggingface.co/datasets/datasets'
UpperCamelCase__						=		'https://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}'

# Metrics
UpperCamelCase__						=		'https://s3.amazonaws.com/datasets.huggingface.co/datasets/metrics'
UpperCamelCase__						=		'https://cdn-datasets.huggingface.co/datasets/metric'
UpperCamelCase__						=		'https://raw.githubusercontent.com/huggingface/datasets/{revision}/metrics/{path}/{name}'

# Hub
UpperCamelCase__						=		os.environ.get('HF_ENDPOINT', 'https://huggingface.co')
UpperCamelCase__						=		HF_ENDPOINT + '/datasets/{repo_id}/resolve/{revision}/{path}'
UpperCamelCase__						=		'main'

UpperCamelCase__						=		version.parse(platform.python_version())

if PY_VERSION < version.parse('3.8'):
							import importlib_metadata
else:
							import importlib.metadata as importlib_metadata

# General environment variables accepted values for booleans
UpperCamelCase__						=		{'1', 'ON', 'YES', 'TRUE'}
UpperCamelCase__						=		ENV_VARS_TRUE_VALUES.union({'AUTO'})


# Imports
UpperCamelCase__						=		version.parse(importlib_metadata.version('dill'))
UpperCamelCase__						=		version.parse(importlib_metadata.version('pandas'))
UpperCamelCase__						=		version.parse(importlib_metadata.version('pyarrow'))

UpperCamelCase__						=		os.environ.get('USE_TF', 'AUTO').upper()
UpperCamelCase__						=		os.environ.get('USE_TORCH', 'AUTO').upper()
UpperCamelCase__						=		os.environ.get('USE_JAX', 'AUTO').upper()

UpperCamelCase__						=		'N/A'
UpperCamelCase__						=		False

if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES:
							UpperCamelCase__						=		importlib.util.find_spec('torch') is not None
							if TORCH_AVAILABLE:
														try:
																					UpperCamelCase__						=		version.parse(importlib_metadata.version('torch'))
																					logger.info(F'PyTorch version {TORCH_VERSION} available.')
														except importlib_metadata.PackageNotFoundError:
																					pass
else:
							logger.info('Disabling PyTorch because USE_TF is set')

UpperCamelCase__						=		'N/A'
UpperCamelCase__						=		False

if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES:
							UpperCamelCase__						=		importlib.util.find_spec('tensorflow') is not None
							if TF_AVAILABLE:
														# For the metadata, we have to look for both tensorflow and tensorflow-cpu
														for package in [
														    "tensorflow",
														    "tensorflow-cpu",
														    "tensorflow-gpu",
														    "tf-nightly",
														    "tf-nightly-cpu",
														    "tf-nightly-gpu",
														    "intel-tensorflow",
														    "tensorflow-rocm",
														    "tensorflow-macos",
														]:
																					try:
																												UpperCamelCase__						=		version.parse(importlib_metadata.version(package))
																					except importlib_metadata.PackageNotFoundError:
																												continue
																					else:
																												break
														else:
																					UpperCamelCase__						=		False
							if TF_AVAILABLE:
														if TF_VERSION.major < 2:
																					logger.info(F'TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.')
																					UpperCamelCase__						=		False
														else:
																					logger.info(F'TensorFlow version {TF_VERSION} available.')
else:
							logger.info('Disabling Tensorflow because USE_TORCH is set')


UpperCamelCase__						=		'N/A'
UpperCamelCase__						=		False

if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES:
							UpperCamelCase__						=		importlib.util.find_spec('jax') is not None and importlib.util.find_spec('jaxlib') is not None
							if JAX_AVAILABLE:
														try:
																					UpperCamelCase__						=		version.parse(importlib_metadata.version('jax'))
																					logger.info(F'JAX version {JAX_VERSION} available.')
														except importlib_metadata.PackageNotFoundError:
																					pass
else:
							logger.info('Disabling JAX because USE_JAX is set to False')


UpperCamelCase__						=		os.environ.get('USE_BEAM', 'AUTO').upper()
UpperCamelCase__						=		'N/A'
UpperCamelCase__						=		False
if USE_BEAM in ENV_VARS_TRUE_AND_AUTO_VALUES:
							try:
														UpperCamelCase__						=		version.parse(importlib_metadata.version('apache_beam'))
														UpperCamelCase__						=		True
														logger.info(F'Apache Beam version {BEAM_VERSION} available.')
							except importlib_metadata.PackageNotFoundError:
														pass
else:
							logger.info('Disabling Apache Beam because USE_BEAM is set to False')


# Optional tools for data loading
UpperCamelCase__						=		importlib.util.find_spec('sqlalchemy') is not None

# Optional tools for feature decoding
UpperCamelCase__						=		importlib.util.find_spec('PIL') is not None
UpperCamelCase__						=		importlib.util.find_spec('soundfile') is not None and version.parse(
    importlib.import_module('soundfile').__libsndfile_version__
) >= version.parse('1.0.31')
UpperCamelCase__						=		importlib.util.find_spec('soundfile') is not None and version.parse(
    importlib.import_module('soundfile').__libsndfile_version__
) >= version.parse('1.1.0')

# Optional compression tools
UpperCamelCase__						=		importlib.util.find_spec('rarfile') is not None
UpperCamelCase__						=		importlib.util.find_spec('zstandard') is not None
UpperCamelCase__						=		importlib.util.find_spec('lz4') is not None
UpperCamelCase__						=		importlib.util.find_spec('py7zr') is not None


# Cache location
UpperCamelCase__						=		'~/.cache'
UpperCamelCase__						=		os.getenv('XDG_CACHE_HOME', DEFAULT_XDG_CACHE_HOME)
UpperCamelCase__						=		os.path.join(XDG_CACHE_HOME, 'huggingface')
UpperCamelCase__						=		os.path.expanduser(os.getenv('HF_HOME', DEFAULT_HF_CACHE_HOME))

UpperCamelCase__						=		os.path.join(HF_CACHE_HOME, 'datasets')
UpperCamelCase__						=		Path(os.getenv('HF_DATASETS_CACHE', DEFAULT_HF_DATASETS_CACHE))

UpperCamelCase__						=		os.path.join(HF_CACHE_HOME, 'metrics')
UpperCamelCase__						=		Path(os.getenv('HF_METRICS_CACHE', DEFAULT_HF_METRICS_CACHE))

UpperCamelCase__						=		os.path.join(HF_CACHE_HOME, 'modules')
UpperCamelCase__						=		Path(os.getenv('HF_MODULES_CACHE', DEFAULT_HF_MODULES_CACHE))

UpperCamelCase__						=		'downloads'
UpperCamelCase__						=		os.path.join(HF_DATASETS_CACHE, DOWNLOADED_DATASETS_DIR)
UpperCamelCase__						=		Path(os.getenv('HF_DATASETS_DOWNLOADED_DATASETS_PATH', DEFAULT_DOWNLOADED_DATASETS_PATH))

UpperCamelCase__						=		'extracted'
UpperCamelCase__						=		os.path.join(DEFAULT_DOWNLOADED_DATASETS_PATH, EXTRACTED_DATASETS_DIR)
UpperCamelCase__						=		Path(os.getenv('HF_DATASETS_EXTRACTED_DATASETS_PATH', DEFAULT_EXTRACTED_DATASETS_PATH))

# Download count for the website
UpperCamelCase__						=		(
    os.environ.get('HF_UPDATE_DOWNLOAD_COUNTS', 'AUTO').upper() in ENV_VARS_TRUE_AND_AUTO_VALUES
)

# Batch size constants. For more info, see:
# https://github.com/apache/arrow/blob/master/docs/source/cpp/arrays.rst#size-limitations-and-recommendations)
UpperCamelCase__						=		1_0_0_0

# Size of the preloaded record batch in `Dataset.__iter__`
UpperCamelCase__						=		1_0

# Pickling tables works only for small tables (<4GiB)
# For big tables, we write them on disk instead
UpperCamelCase__						=		4 << 3_0

# Max shard size in bytes (e.g. to shard parquet datasets in push_to_hub or download_and_prepare)
UpperCamelCase__						=		'500MB'

# Parquet configuration
UpperCamelCase__						=		1_0_0
UpperCamelCase__						=		1_0_0
UpperCamelCase__						=		1_0_0

# Offline mode
UpperCamelCase__						=		os.environ.get('HF_DATASETS_OFFLINE', 'AUTO').upper() in ENV_VARS_TRUE_VALUES

# In-memory
UpperCamelCase__						=		0  # Disabled
UpperCamelCase__						=		float(os.environ.get('HF_DATASETS_IN_MEMORY_MAX_SIZE', DEFAULT_IN_MEMORY_MAX_SIZE))

# File names
UpperCamelCase__						=		'dataset.arrow'
UpperCamelCase__						=		'indices.arrow'
UpperCamelCase__						=		'state.json'
UpperCamelCase__						=		'dataset_info.json'
UpperCamelCase__						=		'dataset_infos.json'
UpperCamelCase__						=		'LICENSE'
UpperCamelCase__						=		'metric_info.json'
UpperCamelCase__						=		'dataset_dict.json'

UpperCamelCase__						=		'datasets_modules'

UpperCamelCase__						=		2_5_5

# Streaming
UpperCamelCase__						=		2_0
UpperCamelCase__						=		5

# Datasets without script
UpperCamelCase__						=		2_0_0
UpperCamelCase__						=		1_0
UpperCamelCase__						=		2_0_0

# Progress bars
UpperCamelCase__						=		0.05  # 20 progress updates per sec


"""simple docstring"""







import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
							import torch

if is_vision_available():
							from PIL import Image

							from transformers import DeiTImageProcessor




class 					A__			(      unittest.TestCase):

		def __init__( self			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE=7			,						_SCREAMING_SNAKE_CASE=3			,						_SCREAMING_SNAKE_CASE=18			,						_SCREAMING_SNAKE_CASE=30			,						_SCREAMING_SNAKE_CASE=4_00			,						_SCREAMING_SNAKE_CASE=True			,						_SCREAMING_SNAKE_CASE=None			,						_SCREAMING_SNAKE_CASE=True			,						_SCREAMING_SNAKE_CASE=None			,						_SCREAMING_SNAKE_CASE=True			,						_SCREAMING_SNAKE_CASE=[0.5, 0.5, 0.5]			,						_SCREAMING_SNAKE_CASE=[0.5, 0.5, 0.5]			,						):
			__lowerCAmelCase     :		List[str]		     = size if size is not None else {'height': 20, 'width': 20}
			__lowerCAmelCase     :		List[str]		     = crop_size if crop_size is not None else {'height': 18, 'width': 18}

			__lowerCAmelCase     :		Any		     = parent
			__lowerCAmelCase     :		Optional[Any]		     = batch_size
			__lowerCAmelCase     :		Any		     = num_channels
			__lowerCAmelCase     :		Dict		     = image_size
			__lowerCAmelCase     :		Tuple		     = min_resolution
			__lowerCAmelCase     :		Union[str, Any]		     = max_resolution
			__lowerCAmelCase     :		Optional[int]		     = do_resize
			__lowerCAmelCase     :		Any		     = size
			__lowerCAmelCase     :		Union[str, Any]		     = do_center_crop
			__lowerCAmelCase     :		List[Any]		     = crop_size
			__lowerCAmelCase     :		Optional[int]		     = do_normalize
			__lowerCAmelCase     :		int		     = image_mean
			__lowerCAmelCase     :		Tuple		     = image_std







		def 				__lowerCamelCase				( self					):
			return {
			    "do_resize": self.do_resize,
			    "size": self.size,
			    "do_center_crop": self.do_center_crop,
			    "crop_size": self.crop_size,
			    "do_normalize": self.do_normalize,
			    "image_mean": self.image_mean,
			    "image_std": self.image_std,
			}




@require_torch
@require_vision
class 					A__			(      _lowerCamelCase			,	unittest.TestCase):
		A_		:			List[str]							  =      DeiTImageProcessor if is_vision_available() else None
		A_		:			int							  =      True

		def 				__lowerCamelCase				( self					):
			__lowerCAmelCase     :		Optional[int]		     = DeiTImageProcessingTester(self					)

		@property
		def 				__lowerCamelCase				( self					):
			return self.image_processor_tester.prepare_image_processor_dict()

		def 				__lowerCamelCase				( self					):
			__lowerCAmelCase     :		Any		     = self.image_processing_class(**self.image_processor_dict					)
			self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE			,						'do_resize'					)					)
			self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE			,						'size'					)					)
			self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE			,						'do_center_crop'					)					)
			self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE			,						'center_crop'					)					)
			self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE			,						'do_normalize'					)					)
			self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE			,						'image_mean'					)					)
			self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE			,						'image_std'					)					)

		def 				__lowerCamelCase				( self					):
			__lowerCAmelCase     :		Tuple		     = self.image_processing_class.from_dict(self.image_processor_dict					)
			self.assertEqual(image_processor.size			,						{'height': 20, 'width': 20}					)
			self.assertEqual(image_processor.crop_size			,						{'height': 18, 'width': 18}					)

			__lowerCAmelCase     :		str		     = self.image_processing_class.from_dict(self.image_processor_dict			,						size=42			,						crop_size=84					)
			self.assertEqual(image_processor.size			,						{'height': 42, 'width': 42}					)
			self.assertEqual(image_processor.crop_size			,						{'height': 84, 'width': 84}					)

		def 				__lowerCamelCase				( self					):
			pass

		def 				__lowerCamelCase				( self					):
			# Initialize image_processing
			__lowerCAmelCase     :		Dict		     = self.image_processing_class(**self.image_processor_dict					)
			# create random PIL images
			__lowerCAmelCase     :		List[str]		     = prepare_image_inputs(self.image_processor_tester			,						equal_resolution=_SCREAMING_SNAKE_CASE					)
			for image in image_inputs:
				self.assertIsInstance(_SCREAMING_SNAKE_CASE			,						Image.Image					)

			# Test not batched input
			__lowerCAmelCase     :		Dict		     = image_processing(image_inputs[0]			,						return_tensors='pt'					).pixel_values
			self.assertEqual(
			    encoded_images.shape			,						(
			        1,
			        self.image_processor_tester.num_channels,
			        self.image_processor_tester.crop_size['height'],
			        self.image_processor_tester.crop_size['width'],
			    )			,						)

			# Test batched
			__lowerCAmelCase     :		int		     = image_processing(_SCREAMING_SNAKE_CASE			,						return_tensors='pt'					).pixel_values
			self.assertEqual(
			    encoded_images.shape			,						(
			        self.image_processor_tester.batch_size,
			        self.image_processor_tester.num_channels,
			        self.image_processor_tester.crop_size['height'],
			        self.image_processor_tester.crop_size['width'],
			    )			,						)

		def 				__lowerCamelCase				( self					):
			# Initialize image_processing
			__lowerCAmelCase     :		Optional[int]		     = self.image_processing_class(**self.image_processor_dict					)
			# create random numpy tensors
			__lowerCAmelCase     :		List[str]		     = prepare_image_inputs(self.image_processor_tester			,						equal_resolution=_SCREAMING_SNAKE_CASE			,						numpify=_SCREAMING_SNAKE_CASE					)
			for image in image_inputs:
				self.assertIsInstance(_SCREAMING_SNAKE_CASE			,						np.ndarray					)

			# Test not batched input
			__lowerCAmelCase     :		str		     = image_processing(image_inputs[0]			,						return_tensors='pt'					).pixel_values
			self.assertEqual(
			    encoded_images.shape			,						(
			        1,
			        self.image_processor_tester.num_channels,
			        self.image_processor_tester.crop_size['height'],
			        self.image_processor_tester.crop_size['width'],
			    )			,						)

			# Test batched
			__lowerCAmelCase     :		Dict		     = image_processing(_SCREAMING_SNAKE_CASE			,						return_tensors='pt'					).pixel_values
			self.assertEqual(
			    encoded_images.shape			,						(
			        self.image_processor_tester.batch_size,
			        self.image_processor_tester.num_channels,
			        self.image_processor_tester.crop_size['height'],
			        self.image_processor_tester.crop_size['width'],
			    )			,						)







		def 				__lowerCamelCase				( self					):
			# Initialize image_processing
			__lowerCAmelCase     :		Union[str, Any]		     = self.image_processing_class(**self.image_processor_dict					)
			# create random PyTorch tensors
			__lowerCAmelCase     :		List[str]		     = prepare_image_inputs(self.image_processor_tester			,						equal_resolution=_SCREAMING_SNAKE_CASE			,						torchify=_SCREAMING_SNAKE_CASE					)
			for image in image_inputs:
				self.assertIsInstance(_SCREAMING_SNAKE_CASE			,						torch.Tensor					)

			# Test not batched input
			__lowerCAmelCase     :		Union[str, Any]		     = image_processing(image_inputs[0]			,						return_tensors='pt'					).pixel_values
			self.assertEqual(
			    encoded_images.shape			,						(
			        1,
			        self.image_processor_tester.num_channels,
			        self.image_processor_tester.crop_size['height'],
			        self.image_processor_tester.crop_size['width'],
			    )			,						)

			# Test batched
			__lowerCAmelCase     :		Optional[int]		     = image_processing(_SCREAMING_SNAKE_CASE			,						return_tensors='pt'					).pixel_values
			self.assertEqual(
			    encoded_images.shape			,						(
			        self.image_processor_tester.batch_size,
			        self.image_processor_tester.num_channels,
			        self.image_processor_tester.crop_size['height'],
			        self.image_processor_tester.crop_size['width'],
			    )			,						)