






import pytest

from datasets.utils.sharding import _distribute_shards, _number_of_shards_in_gen_kwargs, _split_gen_kwargs


@pytest.mark.parametrize(
    'kwargs, expected' ,		[
        ({'num_shards': 0, 'max_num_jobs': 1}, []),
        ({'num_shards': 10, 'max_num_jobs': 1}, [range(10	)]),
        ({'num_shards': 10, 'max_num_jobs': 10}, [range(__lowerCAmelCase ,		i + 1	) for i in range(10	)]),
        ({'num_shards': 1, 'max_num_jobs': 10}, [range(1	)]),
        ({'num_shards': 10, 'max_num_jobs': 3}, [range(0 ,		4	), range(4 ,		7	), range(7 ,		10	)]),
        ({'num_shards': 3, 'max_num_jobs': 10}, [range(0 ,		1	), range(1 ,		2	), range(2 ,		3	)]),
    ] ,		)
def __A    (      __lowerCAmelCase ,		__lowerCAmelCase	)->  Optional[int]:





						"""simple docstring"""



						_UpperCAmelCase									=			_distribute_shards(**__lowerCAmelCase	)
						assert out == expected


@pytest.mark.parametrize(
    'gen_kwargs, max_num_jobs, expected' ,		[
        ({'foo': 0}, 10, [{'foo': 0}]),
        ({'shards': [0, 1, 2, 3]}, 1, [{'shards': [0, 1, 2, 3]}]),
        ({'shards': [0, 1, 2, 3]}, 4, [{'shards': [0]}, {'shards': [1]}, {'shards': [2]}, {'shards': [3]}]),
        ({'shards': [0, 1]}, 4, [{'shards': [0]}, {'shards': [1]}]),
        ({'shards': [0, 1, 2, 3]}, 2, [{'shards': [0, 1]}, {'shards': [2, 3]}]),
    ] ,		)
def __A    (      __lowerCAmelCase ,		__lowerCAmelCase ,		__lowerCAmelCase	)->  Any:





						"""simple docstring"""



						_UpperCAmelCase									=			_split_gen_kwargs(__lowerCAmelCase ,		__lowerCAmelCase	)
						assert out == expected


@pytest.mark.parametrize(
    'gen_kwargs, expected' ,		[
        ({'foo': 0}, 1),
        ({'shards': [0]}, 1),
        ({'shards': [0, 1, 2, 3]}, 4),
        ({'shards': [0, 1, 2, 3], 'foo': 0}, 4),
        ({'shards': [0, 1, 2, 3], 'other': (0, 1)}, 4),
        ({'shards': [0, 1, 2, 3], 'shards2': [0, 1]}, RuntimeError),
    ] ,		)
def __A    (      __lowerCAmelCase ,		__lowerCAmelCase	)->  Dict:





						"""simple docstring"""



						if expected is RuntimeError:
												with pytest.raises(__lowerCAmelCase	):
																		_number_of_shards_in_gen_kwargs(__lowerCAmelCase	)
						else:
												_UpperCAmelCase									=			_number_of_shards_in_gen_kwargs(__lowerCAmelCase	)
												assert out == expected








import enum
import os
from hashlib import shaaaa
from typing import Optional

from .. import config
from .logging import get_logger


_a      	=  get_logger(__name__)





class __lowerCamelCase (      enum.Enum):





	"""simple docstring"""






	UpperCamelCase__								=      "all_checks"
	UpperCamelCase__								=      "basic_checks"
	UpperCamelCase__								=      "no_checks"





class __lowerCamelCase (      snake_case__):





	"""simple docstring"""











class __lowerCamelCase (      snake_case__):





	"""simple docstring"""











class __lowerCamelCase (      snake_case__):





	"""simple docstring"""











class __lowerCamelCase (      snake_case__):





	"""simple docstring"""









def __A    (      __lowerCAmelCase ,		__lowerCAmelCase ,		__lowerCAmelCase=None	)->  str:





						"""simple docstring"""



						if expected_checksums is None:
												logger.info('Unable to verify checksums.'	)
												return
						if len(set(__lowerCAmelCase	) - set(__lowerCAmelCase	)	) > 0:
												raise ExpectedMoreDownloadedFiles(str(set(__lowerCAmelCase	) - set(__lowerCAmelCase	)	)	)
						if len(set(__lowerCAmelCase	) - set(__lowerCAmelCase	)	) > 0:
												raise UnexpectedDownloadedFile(str(set(__lowerCAmelCase	) - set(__lowerCAmelCase	)	)	)
						_UpperCAmelCase									=			[url for url in expected_checksums if expected_checksums[url] != recorded_checksums[url]]
						_UpperCAmelCase									=			' for ' + verification_name if verification_name is not None else ''
						if len(__lowerCAmelCase	) > 0:
												raise NonMatchingChecksumError(
												    F"""Checksums didn't match{for_verification_name}:\n"""
												    F"""{bad_urls}\n"""
												    'Set `verification_mode=\'no_checks\'` to skip checksums verification and ignore this error'	)
						logger.info('All the checksums matched successfully' + for_verification_name	)





class __lowerCamelCase (      snake_case__):





	"""simple docstring"""











class __lowerCamelCase (      snake_case__):





	"""simple docstring"""











class __lowerCamelCase (      snake_case__):





	"""simple docstring"""











class __lowerCamelCase (      snake_case__):





	"""simple docstring"""









def __A    (      __lowerCAmelCase ,		__lowerCAmelCase	)->  int:





						"""simple docstring"""



						if expected_splits is None:
												logger.info('Unable to verify splits sizes.'	)
												return
						if len(set(__lowerCAmelCase	) - set(__lowerCAmelCase	)	) > 0:
												raise ExpectedMoreSplits(str(set(__lowerCAmelCase	) - set(__lowerCAmelCase	)	)	)
						if len(set(__lowerCAmelCase	) - set(__lowerCAmelCase	)	) > 0:
												raise UnexpectedSplits(str(set(__lowerCAmelCase	) - set(__lowerCAmelCase	)	)	)
						_UpperCAmelCase									=			[
						    {'expected': expected_splits[name], 'recorded': recorded_splits[name]}
						    for name in expected_splits
						    if expected_splits[name].num_examples != recorded_splits[name].num_examples
						]
						if len(__lowerCAmelCase	) > 0:
												raise NonMatchingSplitsSizesError(str(__lowerCAmelCase	)	)
						logger.info('All the splits matched successfully.'	)


def __A    (      __lowerCAmelCase ,		__lowerCAmelCase = True	)->  dict:





						"""simple docstring"""



						if record_checksum:
												_UpperCAmelCase									=			shaaaa()
												with open(__lowerCAmelCase ,		'rb'	) as f:
																		for chunk in iter(lambda: f.read(1 << 20	) ,		b''	):
																								m.update(__lowerCAmelCase	)
																		_UpperCAmelCase									=			m.hexdigest()
						else:
												_UpperCAmelCase									=			None
						return {"num_bytes": os.path.getsize(__lowerCAmelCase	), "checksum": checksum}


def __A    (      __lowerCAmelCase	)->  List[str]:





						"""simple docstring"""



						if dataset_size and config.IN_MEMORY_MAX_SIZE:
												return dataset_size < config.IN_MEMORY_MAX_SIZE
						else:
												return False
