


'''simple docstring'''

import os
import sys
from pathlib import Path

import pytest

from datasets import Dataset, IterableDataset
from datasets.distributed import split_dataset_by_node

from .utils import execute_subprocess_async, get_torch_dist_unique_port, require_torch






def __UpperCamelCase (      )   ->  Dict:







     '''simple docstring'''






     _a            =		Dataset.from_dict({"i": range(17					)}					)
     _a            =		len(__lowerCamelCase					)
     _a            =		3
     _a            =		[
         split_dataset_by_node(__lowerCamelCase   ,	rank=__lowerCamelCase   ,	world_size=__lowerCamelCase					) for rank in range(__lowerCamelCase					)
     ]
     assert sum(len(__lowerCamelCase					) for ds in datasets_per_rank					) == full_size
     assert len({tuple(x.values()					) for ds in datasets_per_rank for x in ds}					) == full_size






def __UpperCamelCase (      )   ->  Union[str, Any]:







     '''simple docstring'''






     def gen():
          return ({"i": i} for i in range(17					))

     _a            =		3
     _a            =		IterableDataset.from_generator(__lowerCamelCase					)
     _a            =		len(list(__lowerCamelCase					)					)
     _a            =		[
         split_dataset_by_node(__lowerCamelCase   ,	rank=__lowerCamelCase   ,	world_size=__lowerCamelCase					) for rank in range(__lowerCamelCase					)
     ]
     assert sum(len(list(__lowerCamelCase					)					) for ds in datasets_per_rank					) == full_size
     assert len({tuple(x.values()					) for ds in datasets_per_rank for x in ds}					) == full_size






@pytest.mark.parametrize("shards_per_node"   ,	[1, 2, 3]					)
def __UpperCamelCase (      __lowerCamelCase		:			Optional[Any]					)   ->  Optional[int]:







     '''simple docstring'''






     def gen(__lowerCamelCase		:			List[Any]					):
          for shard in shards:
               yield from ({"i": i, "shard": shard} for i in range(17					))

     _a            =		3
     _a            =		shards_per_node * world_size
     _a            =		{"shards": [F"shard_{shard_idx}.txt" for shard_idx in range(__lowerCamelCase					)]}
     _a            =		IterableDataset.from_generator(__lowerCamelCase   ,	gen_kwargs=__lowerCamelCase					)
     _a            =		len(list(__lowerCamelCase					)					)
     assert full_ds.n_shards == world_size * shards_per_node
     _a            =		[
         split_dataset_by_node(__lowerCamelCase   ,	rank=__lowerCamelCase   ,	world_size=__lowerCamelCase					) for rank in range(__lowerCamelCase					)
     ]
     assert [ds.n_shards for ds in datasets_per_rank] == [shards_per_node] * world_size
     assert sum(len(list(__lowerCamelCase					)					) for ds in datasets_per_rank					) == full_size
     assert len({tuple(x.values()					) for ds in datasets_per_rank for x in ds}					) == full_size






def __UpperCamelCase (      )   ->  int:







     '''simple docstring'''






     def gen():
          return ({"i": i} for i in range(17					))

     _a            =		2
     _a            =		IterableDataset.from_generator(__lowerCamelCase					)
     _a            =		len(list(__lowerCamelCase					)					)

     _a            =		split_dataset_by_node(__lowerCamelCase   ,	rank=0   ,	world_size=__lowerCamelCase					).shuffle(seed=42					)
     assert len(list(__lowerCamelCase					)					) == 1 + full_size // world_size
     with pytest.raises(__lowerCamelCase					):
          split_dataset_by_node(__lowerCamelCase   ,	rank=0   ,	world_size=__lowerCamelCase					).shuffle()

     _a            =		split_dataset_by_node(full_ds.shuffle(seed=42					)   ,	rank=0   ,	world_size=__lowerCamelCase					)
     assert len(list(__lowerCamelCase					)					) == 1 + full_size // world_size
     with pytest.raises(__lowerCamelCase					):
          split_dataset_by_node(full_ds.shuffle()   ,	rank=0   ,	world_size=__lowerCamelCase					)






@pytest.mark.parametrize("streaming"   ,	[False, True]					)
@require_torch
@pytest.mark.skipif(os.name == "nt"   ,	reason="execute_subprocess_async doesn't support windows"					)
@pytest.mark.integration
def __UpperCamelCase (      __lowerCamelCase		:			Optional[int]					)   ->  List[Any]:







     '''simple docstring'''






     _a            =		2
     _a            =		get_torch_dist_unique_port()
     _a            =		Path(__lowerCamelCase					).resolve().parent / "distributed_scripts" / "launch_torch_distributed.py"
     _a            =		F"\n        -m torch.distributed.launch\n        --nproc_per_node={nproc_per_node}\n        --master_port={master_port}\n        {test_script}\n    ".split()
     _a            =		F"\n        --streaming={streaming}\n    ".split()
     _a            =		[sys.executable] + distributed_args + args
     execute_subprocess_async(__lowerCamelCase   ,	env=os.environ.copy()					)






@pytest.mark.parametrize(
    "nproc_per_node, num_workers"   ,	[
        (2, 2),  # each node has 2 shards and each worker has 1 shards
        (3, 2),  # each node uses all the shards but skips examples, and each worker has 2 shards
    ]   ,	)
@require_torch
@pytest.mark.skipif(os.name == "nt"   ,	reason="execute_subprocess_async doesn't support windows"					)
@pytest.mark.integration
def __UpperCamelCase (      __lowerCamelCase		:			str   ,	__lowerCamelCase		:			List[str]					)   ->  Tuple:







     '''simple docstring'''






     _a            =		True
     _a            =		get_torch_dist_unique_port()
     _a            =		Path(__lowerCamelCase					).resolve().parent / "distributed_scripts" / "launch_torch_distributed.py"
     _a            =		F"\n        -m torch.distributed.launch\n        --nproc_per_node={nproc_per_node}\n        --master_port={master_port}\n        {test_script}\n    ".split()
     _a            =		F"\n        --streaming={streaming}\n        --num_workers={num_workers}\n    ".split()
     _a            =		[sys.executable] + distributed_args + args
     execute_subprocess_async(__lowerCamelCase   ,	env=os.environ.copy()					)





'''simple docstring'''

import os
import sys
import time

from . import transposition_cipher as trans_cipher






def __UpperCamelCase (      )   ->  None:







     '''simple docstring'''






     _a            =		"Prehistoric Men.txt"
     _a            =		"Output.txt"
     _a            =		int(input("Enter key: "					)					)
     _a            =		input("Encrypt/Decrypt [e/d]: "					)

     if not os.path.exists(__lowerCamelCase					):
          print(F"File {input_file} does not exist. Quitting..."					)
          sys.exit()
     if os.path.exists(__lowerCamelCase					):
          print(F"Overwrite {output_file}? [y/n]"					)
          _a            =		input("> "					)
          if not response.lower().startswith("y"					):
               sys.exit()

     _a            =		time.time()
     if mode.lower().startswith("e"					):
          with open(__lowerCamelCase					) as f:
               _a            =		f.read()
          _a            =		trans_cipher.encrypt_message(__lowerCamelCase   ,	__lowerCamelCase					)
     elif mode.lower().startswith("d"					):
          with open(__lowerCamelCase					) as f:
               _a            =		f.read()
          _a            =		trans_cipher.decrypt_message(__lowerCamelCase   ,	__lowerCamelCase					)

     with open(__lowerCamelCase   ,	"w"					) as output_obj:
          output_obj.write(__lowerCamelCase					)

     _a            =		round(time.time() - start_time   ,	2					)
     print(("Done (", total_time, "seconds )")					)


if __name__ == "__main__":
   main()

