




'''simple docstring'''






import csv
import json
import os

import datasets


# TODO: Add BibTeX citation
# Find for instance the citation on arxiv or on the dataset repo/website
_lowerCAmelCase       =   '''\
@InProceedings{huggingface:dataset,
title = {A great new dataset},
author={huggingface, Inc.
},
year={2020}
}
'''

# TODO: Add description of the dataset here
# You can copy an official description
_lowerCAmelCase       =   '''\
This new dataset is designed to solve this great NLP task and is crafted with a lot of care.
'''

# TODO: Add a link to an official homepage for the dataset here
_lowerCAmelCase       =   ''''''

# TODO: Add the licence for the dataset here if you can find it
_lowerCAmelCase       =   ''''''

# TODO: Add link to the official dataset URLs here
# The HuggingFace Datasets library doesn't host the datasets but only points to the original files.
# This can be an arbitrary nested dict/list of URLs (see below in `_split_generators` method)
_lowerCAmelCase       =   {
    '''first_domain''': '''https://huggingface.co/great-new-dataset-first_domain.zip''',
    '''second_domain''': '''https://huggingface.co/great-new-dataset-second_domain.zip''',
}



class    A						(					datasets.GeneratorBasedBuilder						):




				'''simple docstring'''





				A       =							datasets.Version("1.1.0"						)

				# This is an example of a dataset with multiple configurations.
				# If you don't want/need to define several sub-sets in your dataset,
				# just remove the BUILDER_CONFIG_CLASS and the BUILDER_CONFIGS attributes.

				# If you need to make complex sub-parts in the datasets with configurable options
				# You can create your own builder configuration class to store attribute, inheriting from datasets.BuilderConfig
				# BUILDER_CONFIG_CLASS = MyBuilderConfig

				# You will be able to load one or the other configurations in the following list with
				# data = datasets.load_dataset('my_dataset', 'first_domain')
				# data = datasets.load_dataset('my_dataset', 'second_domain')
				A       =							[
				    datasets.BuilderConfig(name="first_domain"     ,				version=_lowerCamelCase     ,				description="This part of my dataset covers a first domain"						),
				    datasets.BuilderConfig(name="second_domain"     ,				version=_lowerCamelCase     ,				description="This part of my dataset covers a second domain"						),
				]

				A       =							'first_domain'  # It's not mandatory to have a default configuration. Just use one if it make sense.
				def 			a_			(self     )      ->			Tuple:
							# TODO: This method specifies the datasets.DatasetInfo object which contains informations and typings for the dataset
							if self.config.name == "first_domain":  # This is the name of the configuration selected in BUILDER_CONFIGS above
										__UpperCamelCase :     int										=  datasets.Features(
										    {
										        "sentence": datasets.Value("string"     ),
										        "option1": datasets.Value("string"     ),
										        "answer": datasets.Value("string"     )
										        # These are the features of your dataset like images, labels ...
										    }     )
							else:  # This is an example to show how to have different features for "first_domain" and "second_domain"
										__UpperCamelCase :     Any										=  datasets.Features(
										    {
										        "sentence": datasets.Value("string"     ),
										        "option2": datasets.Value("string"     ),
										        "second_domain_answer": datasets.Value("string"     )
										        # These are the features of your dataset like images, labels ...
										    }     )
							return datasets.DatasetInfo(
							    # This is the description that will appear on the datasets page.
							    description=_DESCRIPTION					,    features=_SCREAMING_SNAKE_CASE					,    homepage=_HOMEPAGE					,    license=_LICENSE					,    citation=_CITATION					,    )
				def 			a_			(self					,    _UpperCAmelCase     )      ->			int:
							# TODO: This method is tasked with downloading/extracting the data and defining the splits depending on the configuration
							# If several configurations are possible (listed in BUILDER_CONFIGS), the configuration selected by the user is in self.config.name

							# dl_manager is a datasets.download.DownloadManager that can be used to download and extract URLS
							# It can accept any type or nested list/dict and will give back the same structure with the url replaced with path to local files.
							# By default the archives will be extracted and a path to a cached folder where they are extracted is returned instead of the archive
							__UpperCamelCase :     Optional[int]										=  _URLS[self.config.name]
							__UpperCamelCase :     str										=  dl_manager.download_and_extract(_SCREAMING_SNAKE_CASE     )
							return [
							    datasets.SplitGenerator(
							        name=datasets.Split.TRAIN					,    gen_kwargs={
							            "filepath": os.path.join(_SCREAMING_SNAKE_CASE					,    "train.jsonl"     ),
							            "split": "train",
							        }					,    ),
							    datasets.SplitGenerator(
							        name=datasets.Split.VALIDATION					,    gen_kwargs={
							            "filepath": os.path.join(_SCREAMING_SNAKE_CASE					,    "dev.jsonl"     ),
							            "split": "dev",
							        }					,    ),
							    datasets.SplitGenerator(
							        name=datasets.Split.TEST					,    gen_kwargs={
							            "filepath": os.path.join(_SCREAMING_SNAKE_CASE					,    "test.jsonl"     ),
							            "split": "test"
							        }					,    ),
							]





				def 			a_			(self					,    _UpperCAmelCase					,    _UpperCAmelCase     )      ->			Dict:
							# TODO: This method handles input defined in _split_generators to yield (key, example) tuples from the dataset.
							# The `key` is for legacy reasons (tfds) and is not important in itself, but must be unique for each example.
							with open(_SCREAMING_SNAKE_CASE					,    encoding="utf-8"     ) as f:
										for key, row in enumerate(_SCREAMING_SNAKE_CASE     ):
													__UpperCamelCase :     Dict										=  json.loads(_SCREAMING_SNAKE_CASE     )
													if self.config.name == "first_domain":
																# Yields examples as (key, example) tuples
																yield key, {
																    "sentence": data["sentence"],
																    "option1": data["option1"],
																    "answer": "" if split == "test" else data["answer"],
																}
													else:
																yield key, {
																    "sentence": data["sentence"],
																    "option2": data["option2"],
																    "second_domain_answer": "" if split == "test" else data["second_domain_answer"],
																}





"""simple docstring"""







import os
import sys
from pathlib import Path

import pytest

from datasets import Dataset, IterableDataset
from datasets.distributed import split_dataset_by_node

from .utils import execute_subprocess_async, get_torch_dist_unique_port, require_torch
def        __lowerCAmelCase							():
	__lowerCAmelCase     :		Optional[int]		     = Dataset.from_dict({'i': range(17						)}						)
	__lowerCAmelCase     :		List[Any]		     = len(_UpperCamelCase						)
	__lowerCAmelCase     :		Tuple		     = 3
	__lowerCAmelCase     :		List[Any]		     = [
	    split_dataset_by_node(_UpperCamelCase			,    rank=_UpperCamelCase			,    world_size=_UpperCamelCase						) for rank in range(_UpperCamelCase						)
	]
	assert sum(len(_UpperCamelCase						) for ds in datasets_per_rank						) == full_size
	assert len({tuple(x.values()						) for ds in datasets_per_rank for x in ds}						) == full_size
def        __lowerCAmelCase							():
	def gen():
		return ({"i": i} for i in range(17						))

	__lowerCAmelCase     :		List[Any]		     = 3
	__lowerCAmelCase     :		Optional[Any]		     = IterableDataset.from_generator(_UpperCamelCase						)
	__lowerCAmelCase     :		List[Any]		     = len(list(_UpperCamelCase						)						)
	__lowerCAmelCase     :		str		     = [
	    split_dataset_by_node(_UpperCamelCase			,    rank=_UpperCamelCase			,    world_size=_UpperCamelCase						) for rank in range(_UpperCamelCase						)
	]
	assert sum(len(list(_UpperCamelCase						)						) for ds in datasets_per_rank						) == full_size
	assert len({tuple(x.values()						) for ds in datasets_per_rank for x in ds}						) == full_size
@pytest.mark.parametrize('shards_per_node'			,    [1, 2, 3]						)
def        __lowerCAmelCase							(_UpperCamelCase						):
	def gen(_UpperCamelCase						):
		for shard in shards:
			yield from ({"i": i, "shard": shard} for i in range(17						))

	__lowerCAmelCase     :		List[Any]		     = 3
	__lowerCAmelCase     :		Optional[int]		     = shards_per_node * world_size
	__lowerCAmelCase     :		Dict		     = {'shards': [F"shard_{shard_idx}.txt" for shard_idx in range(_UpperCamelCase						)]}
	__lowerCAmelCase     :		Dict		     = IterableDataset.from_generator(_UpperCamelCase			,    gen_kwargs=_UpperCamelCase						)
	__lowerCAmelCase     :		str		     = len(list(_UpperCamelCase						)						)
	assert full_ds.n_shards == world_size * shards_per_node
	__lowerCAmelCase     :		Union[str, Any]		     = [
	    split_dataset_by_node(_UpperCamelCase			,    rank=_UpperCamelCase			,    world_size=_UpperCamelCase						) for rank in range(_UpperCamelCase						)
	]
	assert [ds.n_shards for ds in datasets_per_rank] == [shards_per_node] * world_size
	assert sum(len(list(_UpperCamelCase						)						) for ds in datasets_per_rank						) == full_size
	assert len({tuple(x.values()						) for ds in datasets_per_rank for x in ds}						) == full_size
def        __lowerCAmelCase							():
	def gen():
		return ({"i": i} for i in range(17						))

	__lowerCAmelCase     :		str		     = 2
	__lowerCAmelCase     :		Any		     = IterableDataset.from_generator(_UpperCamelCase						)
	__lowerCAmelCase     :		Optional[int]		     = len(list(_UpperCamelCase						)						)

	__lowerCAmelCase     :		Dict		     = split_dataset_by_node(_UpperCamelCase			,    rank=0			,    world_size=_UpperCamelCase						).shuffle(seed=42						)
	assert len(list(_UpperCamelCase						)						) == 1 + full_size // world_size
	with pytest.raises(_UpperCamelCase						):
		split_dataset_by_node(_UpperCamelCase			,    rank=0			,    world_size=_UpperCamelCase						).shuffle()

	__lowerCAmelCase     :		List[str]		     = split_dataset_by_node(full_ds.shuffle(seed=42						)			,    rank=0			,    world_size=_UpperCamelCase						)
	assert len(list(_UpperCamelCase						)						) == 1 + full_size // world_size
	with pytest.raises(_UpperCamelCase						):
		split_dataset_by_node(full_ds.shuffle()			,    rank=0			,    world_size=_UpperCamelCase						)
@pytest.mark.parametrize('streaming'			,    [False, True]						)
@require_torch
@pytest.mark.skipif(os.name == 'nt'			,    reason='execute_subprocess_async doesn\'t support windows'						)
@pytest.mark.integration
def        __lowerCAmelCase							(_UpperCamelCase						):
	__lowerCAmelCase     :		Tuple		     = 2
	__lowerCAmelCase     :		Optional[Any]		     = get_torch_dist_unique_port()
	__lowerCAmelCase     :		List[Any]		     = Path(_UpperCamelCase						).resolve().parent / 'distributed_scripts' / 'launch_torch_distributed.py'
	__lowerCAmelCase     :		int		     = F"\n        -m torch.distributed.launch\n        --nproc_per_node={nproc_per_node}\n        --master_port={master_port}\n        {test_script}\n    ".split()
	__lowerCAmelCase     :		Tuple		     = F"\n        --streaming={streaming}\n    ".split()
	__lowerCAmelCase     :		Optional[Any]		     = [sys.executable] + distributed_args + args
	execute_subprocess_async(_UpperCamelCase			,    env=os.environ.copy()						)
@pytest.mark.parametrize(
    'nproc_per_node, num_workers'			,    [
        (2, 2),  # each node has 2 shards and each worker has 1 shards
        (3, 2),  # each node uses all the shards but skips examples, and each worker has 2 shards
    ]			,    )
@require_torch
@pytest.mark.skipif(os.name == 'nt'			,    reason='execute_subprocess_async doesn\'t support windows'						)
@pytest.mark.integration
def        __lowerCAmelCase							(_UpperCamelCase			,    _UpperCamelCase						):
	__lowerCAmelCase     :		Optional[int]		     = True
	__lowerCAmelCase     :		Optional[Any]		     = get_torch_dist_unique_port()
	__lowerCAmelCase     :		Tuple		     = Path(_UpperCamelCase						).resolve().parent / 'distributed_scripts' / 'launch_torch_distributed.py'
	__lowerCAmelCase     :		Optional[Any]		     = F"\n        -m torch.distributed.launch\n        --nproc_per_node={nproc_per_node}\n        --master_port={master_port}\n        {test_script}\n    ".split()
	__lowerCAmelCase     :		List[Any]		     = F"\n        --streaming={streaming}\n        --num_workers={num_workers}\n    ".split()
	__lowerCAmelCase     :		List[Any]		     = [sys.executable] + distributed_args + args
	execute_subprocess_async(_UpperCamelCase			,    env=os.environ.copy()						)