






"""simple docstring"""



import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
						import torch

if is_vision_available():
						from PIL import Image

						from transformers import DeiTImageProcessor






class 				__SCREAMING_SNAKE_CASE    (					unittest.TestCase		):


							def __init__(      self   :    List[Any]      ,     snake_case   :    Tuple      ,     snake_case   :    List[Any]=7      ,     snake_case   :    Dict=3      ,     snake_case   :    List[str]=18      ,     snake_case   :    Union[str, Any]=30      ,     snake_case   :    Dict=400      ,     snake_case   :    List[str]=True      ,     snake_case   :    Tuple=None      ,     snake_case   :    List[Any]=True      ,     snake_case   :    List[Any]=None      ,     snake_case   :    List[Any]=True      ,     snake_case   :    Optional[Any]=[0.5, 0.5, 0.5]      ,     snake_case   :    str=[0.5, 0.5, 0.5]      ,     ):
													'''simple docstring'''





													A__    :	List[str]			=	size if size is not None else {"""height""": 20, """width""": 20}
													A__    :	Tuple			=	crop_size if crop_size is not None else {"""height""": 18, """width""": 18}

													A__    :	Tuple			=	parent
													A__    :	Dict			=	batch_size
													A__    :	List[str]			=	num_channels
													A__    :	Any			=	image_size
													A__    :	Optional[Any]			=	min_resolution
													A__    :	Any			=	max_resolution
													A__    :	Dict			=	do_resize
													A__    :	int			=	size
													A__    :	int			=	do_center_crop
													A__    :	Any			=	crop_size
													A__    :	Dict			=	do_normalize
													A__    :	Tuple			=	image_mean
													A__    :	Dict			=	image_std


							def     _UpperCamelCase							(      self   :    Tuple				):
													'''simple docstring'''





													return {
													    "do_resize": self.do_resize,
													    "size": self.size,
													    "do_center_crop": self.do_center_crop,
													    "crop_size": self.crop_size,
													    "do_normalize": self.do_normalize,
													    "image_mean": self.image_mean,
													    "image_std": self.image_std,
													}






@require_torch
@require_vision
class 				__SCREAMING_SNAKE_CASE    (					UpperCamelCase							,		unittest.TestCase		):
							snake_case_  =				DeiTImageProcessor if is_vision_available() else None
							snake_case_  =				True


							def     _UpperCamelCase							(      self   :    List[str]				):
													'''simple docstring'''





													A__    :	Optional[int]			=	DeiTImageProcessingTester(self				)


							@property
							def     _UpperCamelCase							(      self   :    str				):
													'''simple docstring'''





													return self.image_processor_tester.prepare_image_processor_dict()


							def     _UpperCamelCase							(      self   :    Optional[int]				):
													'''simple docstring'''





													A__    :	str			=	self.image_processing_class(**self.image_processor_dict				)
													self.assertTrue(hasattr(snake_case      ,     """do_resize"""				)				)
													self.assertTrue(hasattr(snake_case      ,     """size"""				)				)
													self.assertTrue(hasattr(snake_case      ,     """do_center_crop"""				)				)
													self.assertTrue(hasattr(snake_case      ,     """center_crop"""				)				)
													self.assertTrue(hasattr(snake_case      ,     """do_normalize"""				)				)
													self.assertTrue(hasattr(snake_case      ,     """image_mean"""				)				)
													self.assertTrue(hasattr(snake_case      ,     """image_std"""				)				)


							def     _UpperCamelCase							(      self   :    Union[str, Any]				):
													'''simple docstring'''





													A__    :	List[Any]			=	self.image_processing_class.from_dict(self.image_processor_dict				)
													self.assertEqual(image_processor.size      ,     {"""height""": 20, """width""": 20}				)
													self.assertEqual(image_processor.crop_size      ,     {"""height""": 18, """width""": 18}				)

													A__    :	Optional[Any]			=	self.image_processing_class.from_dict(self.image_processor_dict      ,     size=42      ,     crop_size=84				)
													self.assertEqual(image_processor.size      ,     {"""height""": 42, """width""": 42}				)
													self.assertEqual(image_processor.crop_size      ,     {"""height""": 84, """width""": 84}				)


							def     _UpperCamelCase							(      self   :    Tuple				):
													'''simple docstring'''





													pass


							def     _UpperCamelCase							(      self   :    Any				):
													'''simple docstring'''





													A__    :	int			=	self.image_processing_class(**self.image_processor_dict				)
													# create random PIL images
													A__    :	Union[str, Any]			=	prepare_image_inputs(self.image_processor_tester      ,     equal_resolution=snake_case				)
													for image in image_inputs:
																			self.assertIsInstance(snake_case      ,     Image.Image				)

													# Test not batched input
													A__    :	Any			=	image_processing(image_inputs[0]      ,     return_tensors="""pt"""				).pixel_values
													self.assertEqual(
													    encoded_images.shape      ,     (
													        1,
													        self.image_processor_tester.num_channels,
													        self.image_processor_tester.crop_size["""height"""],
													        self.image_processor_tester.crop_size["""width"""],
													    )      ,     )

													# Test batched
													A__    :	Dict			=	image_processing(snake_case      ,     return_tensors="""pt"""				).pixel_values
													self.assertEqual(
													    encoded_images.shape      ,     (
													        self.image_processor_tester.batch_size,
													        self.image_processor_tester.num_channels,
													        self.image_processor_tester.crop_size["""height"""],
													        self.image_processor_tester.crop_size["""width"""],
													    )      ,     )


							def     _UpperCamelCase							(      self   :    Optional[int]				):
													'''simple docstring'''





													A__    :	Optional[int]			=	self.image_processing_class(**self.image_processor_dict				)
													# create random numpy tensors
													A__    :	str			=	prepare_image_inputs(self.image_processor_tester      ,     equal_resolution=snake_case      ,     numpify=snake_case				)
													for image in image_inputs:
																			self.assertIsInstance(snake_case      ,     np.ndarray				)

													# Test not batched input
													A__    :	Optional[Any]			=	image_processing(image_inputs[0]      ,     return_tensors="""pt"""				).pixel_values
													self.assertEqual(
													    encoded_images.shape      ,     (
													        1,
													        self.image_processor_tester.num_channels,
													        self.image_processor_tester.crop_size["""height"""],
													        self.image_processor_tester.crop_size["""width"""],
													    )      ,     )

													# Test batched
													A__    :	Optional[Any]			=	image_processing(snake_case      ,     return_tensors="""pt"""				).pixel_values
													self.assertEqual(
													    encoded_images.shape      ,     (
													        self.image_processor_tester.batch_size,
													        self.image_processor_tester.num_channels,
													        self.image_processor_tester.crop_size["""height"""],
													        self.image_processor_tester.crop_size["""width"""],
													    )      ,     )


							def     _UpperCamelCase							(      self   :    int				):
													'''simple docstring'''





													A__    :	Optional[int]			=	self.image_processing_class(**self.image_processor_dict				)
													# create random PyTorch tensors
													A__    :	Any			=	prepare_image_inputs(self.image_processor_tester      ,     equal_resolution=snake_case      ,     torchify=snake_case				)
													for image in image_inputs:
																			self.assertIsInstance(snake_case      ,     torch.Tensor				)

													# Test not batched input
													A__    :	str			=	image_processing(image_inputs[0]      ,     return_tensors="""pt"""				).pixel_values
													self.assertEqual(
													    encoded_images.shape      ,     (
													        1,
													        self.image_processor_tester.num_channels,
													        self.image_processor_tester.crop_size["""height"""],
													        self.image_processor_tester.crop_size["""width"""],
													    )      ,     )

													# Test batched
													A__    :	List[Any]			=	image_processing(snake_case      ,     return_tensors="""pt"""				).pixel_values
													self.assertEqual(
													    encoded_images.shape      ,     (
													        self.image_processor_tester.batch_size,
													        self.image_processor_tester.num_channels,
													        self.image_processor_tester.crop_size["""height"""],
													        self.image_processor_tester.crop_size["""width"""],
													    )      ,     )














"""simple docstring"""



import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


A_     =				get_logger(__name__)


if TYPE_CHECKING:
						from .builder import DatasetBuilder




def   _lowerCAmelCase		(				UpperCAmelCase__							:       Optional[int],    UpperCAmelCase__							:       Optional[Union[str, bool]] = None   )		->Optional[Any]:
						A__    :	int			=	importlib.import_module(UpperCAmelCase__   )

						# TODO(QL): always update the module to add subsequent new authentication
						if hasattr(UpperCAmelCase__,    """_patched_for_streaming"""   ) and module._patched_for_streaming:
												return

						def wrap_auth(UpperCAmelCase__							:       List[str]   ):
												@wraps(UpperCAmelCase__   )
												def wrapper(*UpperCAmelCase__							:       Dict,    **UpperCAmelCase__							:       Dict   ):
																		return function(*UpperCAmelCase__,    use_auth_token=UpperCAmelCase__,    **UpperCAmelCase__   )

												A__    :	List[Any]			=	"""wrap_auth"""
												return wrapper

						# open files in a streaming fashion
						patch_submodule(UpperCAmelCase__,    """open""",    wrap_auth(UpperCAmelCase__   )   ).start()
						patch_submodule(UpperCAmelCase__,    """os.listdir""",    wrap_auth(UpperCAmelCase__   )   ).start()
						patch_submodule(UpperCAmelCase__,    """os.walk""",    wrap_auth(UpperCAmelCase__   )   ).start()
						patch_submodule(UpperCAmelCase__,    """glob.glob""",    wrap_auth(UpperCAmelCase__   )   ).start()
						# allow to navigate in remote zip files
						patch_submodule(UpperCAmelCase__,    """os.path.join""",    UpperCAmelCase__   ).start()
						patch_submodule(UpperCAmelCase__,    """os.path.dirname""",    UpperCAmelCase__   ).start()
						patch_submodule(UpperCAmelCase__,    """os.path.basename""",    UpperCAmelCase__   ).start()
						patch_submodule(UpperCAmelCase__,    """os.path.relpath""",    UpperCAmelCase__   ).start()
						patch_submodule(UpperCAmelCase__,    """os.path.split""",    UpperCAmelCase__   ).start()
						patch_submodule(UpperCAmelCase__,    """os.path.splitext""",    UpperCAmelCase__   ).start()
						# allow checks on paths
						patch_submodule(UpperCAmelCase__,    """os.path.exists""",    wrap_auth(UpperCAmelCase__   )   ).start()
						patch_submodule(UpperCAmelCase__,    """os.path.isdir""",    wrap_auth(UpperCAmelCase__   )   ).start()
						patch_submodule(UpperCAmelCase__,    """os.path.isfile""",    wrap_auth(UpperCAmelCase__   )   ).start()
						patch_submodule(UpperCAmelCase__,    """os.path.getsize""",    wrap_auth(UpperCAmelCase__   )   ).start()
						patch_submodule(UpperCAmelCase__,    """pathlib.Path""",    UpperCAmelCase__   ).start()
						# file readers
						patch_submodule(UpperCAmelCase__,    """gzip.open""",    wrap_auth(UpperCAmelCase__   )   ).start()
						patch_submodule(UpperCAmelCase__,    """numpy.load""",    wrap_auth(UpperCAmelCase__   )   ).start()
						patch_submodule(UpperCAmelCase__,    """pandas.read_csv""",    wrap_auth(UpperCAmelCase__   ),    attrs=["""__version__"""]   ).start()
						patch_submodule(UpperCAmelCase__,    """pandas.read_excel""",    wrap_auth(UpperCAmelCase__   ),    attrs=["""__version__"""]   ).start()
						patch_submodule(UpperCAmelCase__,    """scipy.io.loadmat""",    wrap_auth(UpperCAmelCase__   ),    attrs=["""__version__"""]   ).start()
						patch_submodule(UpperCAmelCase__,    """xml.etree.ElementTree.parse""",    wrap_auth(UpperCAmelCase__   )   ).start()
						patch_submodule(UpperCAmelCase__,    """xml.dom.minidom.parse""",    wrap_auth(UpperCAmelCase__   )   ).start()
						A__    :	Any			=	True




def   _lowerCAmelCase		(				UpperCAmelCase__							:       "DatasetBuilder"   )		->Tuple:
						extend_module_for_streaming(builder.__module__,    use_auth_token=builder.use_auth_token   )
						# if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
						if not builder.__module__.startswith("""datasets."""   ):  # check that it's not a packaged builder like csv
												for imports in get_imports(inspect.getfile(builder.__class__   )   ):
																		if imports[0] == "internal":
																								A__    :	Any			=	imports[1]
																								A__    :	Any			=	""".""".join(builder.__module__.split("""."""   )[:-1] + [internal_import_name]   )
																								extend_module_for_streaming(UpperCAmelCase__,    use_auth_token=builder.use_auth_token   )

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
						from .builder import DatasetBuilder

						A__    :	Any			=	[
						    cls.__module__
						    for cls in type(UpperCAmelCase__   ).__mro__[1:]  # make sure it's not the same module we've already patched
						    if issubclass(cls,    UpperCAmelCase__   ) and cls.__module__ != DatasetBuilder.__module__
						]  # check it's not a standard builder from datasets.builder
						for module in parent_builder_modules:
												extend_module_for_streaming(UpperCAmelCase__,    use_auth_token=builder.use_auth_token   )






