




'''simple docstring'''






import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader



class    A						(					SCREAMING_SNAKE_CASE__						):




				'''simple docstring'''





				def __init__(self					,    _UpperCAmelCase					,    _UpperCAmelCase = None					,    _UpperCAmelCase = None					,    _UpperCAmelCase = None					,    _UpperCAmelCase = False					,    _UpperCAmelCase = False					,    _UpperCAmelCase = None					,    **_UpperCAmelCase					,    )      ->			List[str]:
							super().__init__(
							    _UpperCAmelCase					,    split=_UpperCAmelCase					,    features=_UpperCAmelCase					,    cache_dir=_UpperCAmelCase					,    keep_in_memory=_UpperCAmelCase					,    streaming=_UpperCAmelCase					,    num_proc=_UpperCAmelCase					,    **_UpperCAmelCase					,    )
							__UpperCamelCase :     Dict										=  path_or_paths if isinstance(_UpperCAmelCase					,    _UpperCAmelCase     ) else {self.split: path_or_paths}
							__UpperCamelCase :     str										=  Csv(
							    cache_dir=_UpperCAmelCase					,    data_files=_UpperCAmelCase					,    features=_UpperCAmelCase					,    **_UpperCAmelCase					,    )





				def 			a_			(self     )      ->			Optional[int]:
							# Build iterable dataset
							if self.streaming:
										__UpperCamelCase :     Union[str, Any]										=  self.builder.as_streaming_dataset(split=self.split     )
							# Build regular (map-style) dataset
							else:
										__UpperCamelCase :     Optional[Any]										=  None
										__UpperCamelCase :     Optional[Any]										=  None
										__UpperCamelCase :     Optional[int]										=  None
										__UpperCamelCase :     Union[str, Any]										=  None

										self.builder.download_and_prepare(
										    download_config=_UpperCAmelCase					,    download_mode=_UpperCAmelCase					,    verification_mode=_UpperCAmelCase					,    base_path=_UpperCAmelCase					,    num_proc=self.num_proc					,    )
										__UpperCamelCase :     List[str]										=  self.builder.as_dataset(
										    split=self.split					,    verification_mode=_UpperCAmelCase					,    in_memory=self.keep_in_memory     )
							return dataset







class    A						:




				'''simple docstring'''





				def __init__(self					,    _UpperCAmelCase					,    _UpperCAmelCase					,    _UpperCAmelCase = None					,    _UpperCAmelCase = None					,    **_UpperCAmelCase					,    )      ->			int:
							if num_proc is not None and num_proc <= 0:
										raise ValueError(f"num_proc {num_proc} must be an integer > 0."     )

							__UpperCamelCase :     str										=  dataset
							__UpperCamelCase :     Union[str, Any]										=  path_or_buf
							__UpperCamelCase :     Union[str, Any]										=  batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
							__UpperCamelCase :     Optional[Any]										=  num_proc
							__UpperCamelCase :     Dict										=  "utf-8"
							__UpperCamelCase :     str										=  to_csv_kwargs
				def 			a_			(self     )      ->			int:
							__UpperCamelCase :     Optional[Any]										=  self.to_csv_kwargs.pop("path_or_buf"					,    _UpperCAmelCase     )
							__UpperCamelCase :     Any										=  self.to_csv_kwargs.pop("header"					,    _UpperCAmelCase     )
							__UpperCamelCase :     Optional[int]										=  self.to_csv_kwargs.pop("index"					,    _UpperCAmelCase     )

							if isinstance(self.path_or_buf					,    (str, bytes, os.PathLike)     ):
										with open(self.path_or_buf					,    "wb+"     ) as buffer:
													__UpperCamelCase :     Union[str, Any]										=  self._write(file_obj=_UpperCAmelCase					,    header=_UpperCAmelCase					,    index=_UpperCAmelCase					,    **self.to_csv_kwargs     )
							else:
										__UpperCamelCase :     str										=  self._write(file_obj=self.path_or_buf					,    header=_UpperCAmelCase					,    index=_UpperCAmelCase					,    **self.to_csv_kwargs     )
							return written
				def 			a_			(self					,    _UpperCAmelCase     )      ->			str:
							__UpperCamelCase						,   __UpperCamelCase						,   __UpperCamelCase						,   __UpperCamelCase :     str										=  args

							__UpperCamelCase :     List[Any]										=  query_table(
							    table=self.dataset.data					,    key=slice(_UpperCAmelCase					,    offset + self.batch_size     )					,    indices=self.dataset._indices					,    )
							__UpperCamelCase :     Optional[int]										=  batch.to_pandas().to_csv(
							    path_or_buf=_UpperCAmelCase					,    header=header if (offset == 0) else False					,    index=_UpperCAmelCase					,    **_UpperCAmelCase     )
							return csv_str.encode(self.encoding     )





				def 			a_			(self					,    _UpperCAmelCase					,    _UpperCAmelCase					,    _UpperCAmelCase					,    **_UpperCAmelCase     )      ->			int:
							__UpperCamelCase :     Optional[int]										=  0

							if self.num_proc is None or self.num_proc == 1:
										for offset in logging.tqdm(
										    range(0					,    len(self.dataset     )					,    self.batch_size     )					,    unit="ba"					,    disable=not logging.is_progress_bar_enabled()					,    desc="Creating CSV from Arrow format"					,    ):
													__UpperCamelCase :     Dict										=  self._batch_csv((offset, header, index, to_csv_kwargs)     )
													written += file_obj.write(_UpperCAmelCase     )

							else:
										__UpperCamelCase						,   __UpperCamelCase :     Optional[Any]										=  len(self.dataset     ), self.batch_size
										with multiprocessing.Pool(self.num_proc     ) as pool:
													for csv_str in logging.tqdm(
													    pool.imap(
													        self._batch_csv					,    [(offset, header, index, to_csv_kwargs) for offset in range(0					,    _UpperCAmelCase					,    _UpperCAmelCase     )]					,    )					,    total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size					,    unit="ba"					,    disable=not logging.is_progress_bar_enabled()					,    desc="Creating CSV from Arrow format"					,    ):
																written += file_obj.write(_UpperCAmelCase     )

							return written









'''simple docstring'''






import os
import pickle
import unittest
from collections import UserDict, namedtuple

import torch

from accelerate.test_utils.testing import require_cuda, require_torch_min_version
from accelerate.test_utils.training import RegressionModel
from accelerate.utils import (
    convert_outputs_to_fpaa,
    extract_model_from_parallel,
    find_device,
    listify,
    patch_environment,
    recursively_apply,
    send_to_device,
)


_lowerCAmelCase       =   namedtuple('''ExampleNamedTuple''', '''a b c''')



class    A						(					unittest.TestCase						):




				'''simple docstring'''





				def 			a_			(self     )      ->			Tuple:
							__UpperCamelCase :     Optional[int]										=  torch.randn(5					,    2     )
							__UpperCamelCase :     List[str]										=  torch.device("cuda"     ) if torch.cuda.is_available() else torch.device("cpu"     )

							__UpperCamelCase :     Optional[Any]										=  send_to_device(_UpperCAmelCase					,    _UpperCAmelCase     )
							self.assertTrue(torch.equal(resulta.cpu()					,    _UpperCAmelCase     )     )

							__UpperCamelCase :     Any										=  send_to_device((tensor, [tensor, tensor], 1)					,    _UpperCAmelCase     )
							self.assertIsInstance(_UpperCAmelCase					,    _UpperCAmelCase     )
							self.assertTrue(torch.equal(resulta[0].cpu()					,    _UpperCAmelCase     )     )
							self.assertIsInstance(resulta[1]					,    _UpperCAmelCase     )
							self.assertTrue(torch.equal(resulta[1][0].cpu()					,    _UpperCAmelCase     )     )
							self.assertTrue(torch.equal(resulta[1][1].cpu()					,    _UpperCAmelCase     )     )
							self.assertEqual(resulta[2]					,    1     )

							__UpperCamelCase :     Optional[int]										=  send_to_device({"a": tensor, "b": [tensor, tensor], "c": 1}					,    _UpperCAmelCase     )
							self.assertIsInstance(_UpperCAmelCase					,    _UpperCAmelCase     )
							self.assertTrue(torch.equal(resulta["a"].cpu()					,    _UpperCAmelCase     )     )
							self.assertIsInstance(resulta["b"]					,    _UpperCAmelCase     )
							self.assertTrue(torch.equal(resulta["b"][0].cpu()					,    _UpperCAmelCase     )     )
							self.assertTrue(torch.equal(resulta["b"][1].cpu()					,    _UpperCAmelCase     )     )
							self.assertEqual(resulta["c"]					,    1     )

							__UpperCamelCase :     Optional[Any]										=  send_to_device(ExampleNamedTuple(a=_UpperCAmelCase					,    b=[tensor, tensor]					,    c=1     )					,    _UpperCAmelCase     )
							self.assertIsInstance(_UpperCAmelCase					,    _UpperCAmelCase     )
							self.assertTrue(torch.equal(resulta.a.cpu()					,    _UpperCAmelCase     )     )
							self.assertIsInstance(resulta.b					,    _UpperCAmelCase     )
							self.assertTrue(torch.equal(resulta.b[0].cpu()					,    _UpperCAmelCase     )     )
							self.assertTrue(torch.equal(resulta.b[1].cpu()					,    _UpperCAmelCase     )     )
							self.assertEqual(resulta.c					,    1     )

							__UpperCamelCase :     Dict										=  send_to_device(UserDict({"a": tensor, "b": [tensor, tensor], "c": 1}     )					,    _UpperCAmelCase     )
							self.assertIsInstance(_UpperCAmelCase					,    _UpperCAmelCase     )
							self.assertTrue(torch.equal(resulta["a"].cpu()					,    _UpperCAmelCase     )     )
							self.assertIsInstance(resulta["b"]					,    _UpperCAmelCase     )
							self.assertTrue(torch.equal(resulta["b"][0].cpu()					,    _UpperCAmelCase     )     )
							self.assertTrue(torch.equal(resulta["b"][1].cpu()					,    _UpperCAmelCase     )     )
							self.assertEqual(resulta["c"]					,    1     )
				def 			a_			(self     )      ->			Optional[Any]:
							with self.assertRaises(_UpperCAmelCase     ) as cm:
										__UpperCamelCase :     int										=  recursively_apply(torch.tensor					,    (torch.tensor(1     ), 1)					,    error_on_other_type=_UpperCAmelCase     )
							self.assertEqual(
							    str(cm.exception     )					,    "Unsupported types (<class 'int'>) passed to `tensor`. Only nested list/tuple/dicts of objects that are valid for `is_torch_tensor` should be passed."					,    )
				def 			a_			(self     )      ->			Union[str, Any]:
							__UpperCamelCase :     Dict										=  torch.tensor([1, 2, 3, 4, 5]     )
							self.assertEqual(listify(_UpperCAmelCase     )					,    [1, 2, 3, 4, 5]     )

							__UpperCamelCase :     int										=  torch.tensor([[1, 2, 3, 4, 5], [6, 7, 8, 9, 1_0]]     )
							self.assertEqual(listify(_UpperCAmelCase     )					,    [[1, 2, 3, 4, 5], [6, 7, 8, 9, 1_0]]     )

							__UpperCamelCase :     Dict										=  torch.tensor([[[1, 2, 3, 4, 5], [6, 7, 8, 9, 1_0]], [[1_1, 1_2, 1_3, 1_4, 1_5], [1_6, 1_7, 1_8, 1_9, 2_0]]]     )
							self.assertEqual(
							    listify(_UpperCAmelCase     )					,    [[[1, 2, 3, 4, 5], [6, 7, 8, 9, 1_0]], [[1_1, 1_2, 1_3, 1_4, 1_5], [1_6, 1_7, 1_8, 1_9, 2_0]]]     )
				def 			a_			(self     )      ->			Optional[Any]:
							with patch_environment(aa=1					,    BB=2     ):
										self.assertEqual(os.environ.get("AA"     )					,    "1"     )
										self.assertEqual(os.environ.get("BB"     )					,    "2"     )

							self.assertNotIn("AA"					,    os.environ     )
							self.assertNotIn("BB"					,    os.environ     )
				def 			a_			(self     )      ->			Dict:
							__UpperCamelCase :     Tuple										=  RegressionModel()
							__UpperCamelCase :     List[str]										=  model.forward
							__UpperCamelCase :     Any										=  convert_outputs_to_fpaa(model.forward     )
							__UpperCamelCase :     List[str]										=  extract_model_from_parallel(_UpperCAmelCase					,    keep_fpaa_wrapper=_UpperCAmelCase     )
							__UpperCamelCase :     Dict										=  pickle.dumps(_UpperCAmelCase     )
				@require_cuda
				def 			a_			(self     )      ->			str:
							__UpperCamelCase :     List[str]										=  RegressionModel()
							__UpperCamelCase :     List[Any]										=  model.forward
							__UpperCamelCase :     List[str]										=  torch.cuda.amp.autocast(dtype=torch.floataa     )(model.forward     )
							__UpperCamelCase :     str										=  convert_outputs_to_fpaa(model.forward     )
							__UpperCamelCase :     int										=  extract_model_from_parallel(_UpperCAmelCase					,    keep_fpaa_wrapper=_UpperCAmelCase     )
							__UpperCamelCase :     int										=  pickle.dumps(_UpperCAmelCase     )
				@require_cuda
				@require_torch_min_version(version="2.0"     )
				def 			a_			(self     )      ->			Tuple:
							__UpperCamelCase :     List[Any]										=  RegressionModel()
							__UpperCamelCase :     Tuple										=  model.forward
							__UpperCamelCase :     Optional[int]										=  torch.cuda.amp.autocast(dtype=torch.floataa     )(model.forward     )
							__UpperCamelCase :     List[Any]										=  convert_outputs_to_fpaa(model.forward     )
							__UpperCamelCase :     Union[str, Any]										=  torch.compile(model.forward					,    backend="inductor"     )
							__UpperCamelCase :     int										=  torch.randn(4					,    1_0     ).cuda()
							__UpperCamelCase :     int										=  model(_UpperCAmelCase     )
				def 			a_			(self     )      ->			Union[str, Any]:
							__UpperCamelCase :     List[Any]										=  RegressionModel()
							# could also do a test with DistributedDataParallel, but difficult to run on CPU or single GPU
							__UpperCamelCase :     Optional[Any]										=  torch.nn.parallel.DataParallel(_UpperCAmelCase     )
							__UpperCamelCase :     Union[str, Any]										=  extract_model_from_parallel(_UpperCAmelCase     )

							self.assertEqual(_UpperCAmelCase					,    _UpperCAmelCase     )
				@require_torch_min_version(version="2.0"     )
				def 			a_			(self     )      ->			Union[str, Any]:
							__UpperCamelCase :     int										=  RegressionModel()
							__UpperCamelCase :     List[str]										=  torch.compile(_UpperCAmelCase     )

							# could also do a test with DistributedDataParallel, but difficult to run on CPU or single GPU
							__UpperCamelCase :     Union[str, Any]										=  torch.nn.parallel.DataParallel(_UpperCAmelCase     )
							__UpperCamelCase :     List[str]										=  torch.compile(_UpperCAmelCase     )
							__UpperCamelCase :     Optional[int]										=  extract_model_from_parallel(_UpperCAmelCase     )

							self.assertEqual(compiled_model._orig_mod					,    compiled_model_unwrapped._orig_mod     )





				def 			a_			(self     )      ->			str:
							self.assertEqual(find_device([1, "a", torch.tensor([1, 2, 3]     )]     )					,    torch.device("cpu"     )     )
							self.assertEqual(find_device({"a": 1, "b": torch.tensor([1, 2, 3]     )}     )					,    torch.device("cpu"     )     )
							self.assertIsNone(find_device([1, "a"]     )     )



