
import operator as op


__lowerCamelCase							=       """scaler.pt"""
__lowerCamelCase							=       """pytorch_model"""
__lowerCamelCase							=       """random_states"""
__lowerCamelCase							=       """optimizer"""
__lowerCamelCase							=       """scheduler"""
__lowerCamelCase							=       """pytorch_model.bin"""
__lowerCamelCase							=       """pytorch_model.bin.index.json"""
__lowerCamelCase							=       """model.safetensors"""
__lowerCamelCase							=       """model.safetensors.index.json"""
__lowerCamelCase							=       """1.10.2"""
__lowerCamelCase							=       """py38"""
__lowerCamelCase							=       """4.17.0"""
__lowerCamelCase							=       ["""ml.p3.16xlarge""", """ml.p3dn.24xlarge""", """ml.p4dn.24xlarge"""]
__lowerCamelCase							=       ["""FULL_SHARD""", """SHARD_GRAD_OP""", """NO_SHARD""", """HYBRID_SHARD""", """HYBRID_SHARD_ZERO2"""]
__lowerCamelCase							=       ["""TRANSFORMER_BASED_WRAP""", """SIZE_BASED_WRAP""", """NO_WRAP"""]
__lowerCamelCase							=       ["""BACKWARD_PRE""", """BACKWARD_POST""", """NO_PREFETCH"""]
__lowerCamelCase							=       ["""FULL_STATE_DICT""", """LOCAL_STATE_DICT""", """SHARDED_STATE_DICT"""]
__lowerCamelCase							=       """2.0.1"""
__lowerCamelCase							=       ["""pdsh""", """standard""", """openmpi""", """mvapich"""]
__lowerCamelCase							=       ["""default""", """reduce-overhead""", """max-autotune"""]

__lowerCamelCase							=       {""">""": op.gt, """>=""": op.ge, """==""": op.eq, """!=""": op.ne, """<=""": op.le, """<""": op.lt}

# These are the args for `torch.distributed.launch` for pytorch < 1.9
__lowerCamelCase							=       [
    """nnodes""",
    """nproc_per_node""",
    """rdzv_backend""",
    """rdzv_endpoint""",
    """rdzv_id""",
    """rdzv_conf""",
    """standalone""",
    """max_restarts""",
    """monitor_interval""",
    """start_method""",
    """role""",
    """module""",
    """m""",
    """no_python""",
    """run_path""",
    """log_dir""",
    """r""",
    """redirects""",
    """t""",
    """tee""",
    """node_rank""",
    """master_addr""",
    """master_port""",
]

__lowerCamelCase							=       ["""DEEPSPEED""", """MULTI_GPU""", """FSDP""", """MEGATRON_LM"""]
__lowerCamelCase							=       ["""DEEPSPEED""", """MULTI_XPU""", """FSDP"""]









"""simple docstring"""







import pytest

from datasets.utils.sharding import _distribute_shards, _number_of_shards_in_gen_kwargs, _split_gen_kwargs




@pytest.mark.parametrize(
    "kwargs, expected"  ,						[
        ({"num_shards": 0, "max_num_jobs": 1}, []),
        ({"num_shards": 10, "max_num_jobs": 1}, [range(10   )]),
        ({"num_shards": 10, "max_num_jobs": 10}, [range(lowerCAmelCase_  ,						i + 1   ) for i in range(10   )]),
        ({"num_shards": 1, "max_num_jobs": 10}, [range(1   )]),
        ({"num_shards": 10, "max_num_jobs": 3}, [range(0  ,						4   ), range(4  ,						7   ), range(7  ,						10   )]),
        ({"num_shards": 3, "max_num_jobs": 10}, [range(0  ,						1   ), range(1  ,						2   ), range(2  ,						3   )]),
    ]  ,						)
def    UpperCAmelCase__       (lowerCAmelCase_  ,						lowerCAmelCase_   ):




					'''simple docstring'''


					__SCREAMING_SNAKE_CASE						   =				_distribute_shards(**lowerCAmelCase_   )
					assert out == expected




@pytest.mark.parametrize(
    "gen_kwargs, max_num_jobs, expected"  ,						[
        ({"foo": 0}, 10, [{"foo": 0}]),
        ({"shards": [0, 1, 2, 3]}, 1, [{"shards": [0, 1, 2, 3]}]),
        ({"shards": [0, 1, 2, 3]}, 4, [{"shards": [0]}, {"shards": [1]}, {"shards": [2]}, {"shards": [3]}]),
        ({"shards": [0, 1]}, 4, [{"shards": [0]}, {"shards": [1]}]),
        ({"shards": [0, 1, 2, 3]}, 2, [{"shards": [0, 1]}, {"shards": [2, 3]}]),
    ]  ,						)
def    UpperCAmelCase__       (lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_   ):




					'''simple docstring'''


					__SCREAMING_SNAKE_CASE						   =				_split_gen_kwargs(lowerCAmelCase_  ,						lowerCAmelCase_   )
					assert out == expected




@pytest.mark.parametrize(
    "gen_kwargs, expected"  ,						[
        ({"foo": 0}, 1),
        ({"shards": [0]}, 1),
        ({"shards": [0, 1, 2, 3]}, 4),
        ({"shards": [0, 1, 2, 3], "foo": 0}, 4),
        ({"shards": [0, 1, 2, 3], "other": (0, 1)}, 4),
        ({"shards": [0, 1, 2, 3], "shards2": [0, 1]}, RuntimeError),
    ]  ,						)
def    UpperCAmelCase__       (lowerCAmelCase_  ,						lowerCAmelCase_   ):




					'''simple docstring'''


					if expected is RuntimeError:
										with pytest.raises(lowerCAmelCase_   ):
															_number_of_shards_in_gen_kwargs(lowerCAmelCase_   )
					else:
										__SCREAMING_SNAKE_CASE						   =				_number_of_shards_in_gen_kwargs(lowerCAmelCase_   )
										assert out == expected
