




'''simple docstring'''

import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
							import sqlitea

							import sqlalchemy






class 						a				(   _lowerCamelCase     ):


				def __init__(		self    :					str       ,				lowercase_    :					Union[str, "sqlalchemy.sql.Selectable"]       ,				lowercase_    :					Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"]       ,				lowercase_    :					Optional[Features] = None       ,				lowercase_    :					str = None       ,				lowercase_    :					bool = False       ,				**lowercase_    :					Optional[int]       ,				):
									super().__init__(features=lowercase_       ,				cache_dir=lowercase_       ,				keep_in_memory=lowercase_       ,				**lowercase_					)
									snake_case_									=				Sql(
									    cache_dir=lowercase_       ,				features=lowercase_       ,				sql=lowercase_       ,				con=lowercase_       ,				**lowercase_       ,				)


				def   A_ (		self    :					List[Any]					):
									snake_case_									=				None
									snake_case_									=				None
									snake_case_									=				None
									snake_case_									=				None

									self.builder.download_and_prepare(
									    download_config=lowercase_       ,				download_mode=lowercase_       ,				verification_mode=lowercase_       ,				base_path=lowercase_       ,				)

									# Build dataset for splits
									snake_case_									=				self.builder.as_dataset(
									    split='''train'''       ,				verification_mode=lowercase_       ,				in_memory=self.keep_in_memory					)
									return dataset







class 						a				:


				def __init__(		self    :					Tuple       ,				lowercase_    :					Dataset       ,				lowercase_    :					str       ,				lowercase_    :					Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"]       ,				lowercase_    :					Optional[int] = None       ,				lowercase_    :					Optional[int] = None       ,				**lowercase_    :					Optional[Any]       ,				):
									if num_proc is not None and num_proc <= 0:
														raise ValueError(F"num_proc {num_proc} must be an integer > 0."					)

									snake_case_									=				dataset
									snake_case_									=				name
									snake_case_									=				con
									snake_case_									=				batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
									snake_case_									=				num_proc
									snake_case_									=				to_sql_kwargs


				def   A_ (		self    :					str					):
									snake_case_									=				self.to_sql_kwargs.pop('''sql'''       ,				lowercase_					)
									snake_case_									=				self.to_sql_kwargs.pop('''con'''       ,				lowercase_					)
									snake_case_									=				self.to_sql_kwargs.pop('''index'''       ,				lowercase_					)

									snake_case_									=				self._write(index=lowercase_       ,				**self.to_sql_kwargs					)
									return written


				def   A_ (		self    :					List[str]       ,				lowercase_    :					Any					):
									snake_case_			,snake_case_			,snake_case_									=				args
									snake_case_									=				{**to_sql_kwargs, '''if_exists''': '''append'''} if offset > 0 else to_sql_kwargs
									snake_case_									=				query_table(
									    table=self.dataset.data       ,				key=slice(lowercase_       ,				offset + self.batch_size					)       ,				indices=self.dataset._indices       ,				)
									snake_case_									=				batch.to_pandas()
									snake_case_									=				df.to_sql(self.name       ,				self.con       ,				index=lowercase_       ,				**lowercase_					)
									return num_rows or len(lowercase_					)


				def   A_ (		self    :					Optional[int]       ,				lowercase_    :					Optional[int]       ,				**lowercase_    :					Union[str, Any]					):
									snake_case_									=				0

									if self.num_proc is None or self.num_proc == 1:
														for offset in logging.tqdm(
														    range(0       ,				len(self.dataset					)       ,				self.batch_size					)       ,				unit='''ba'''       ,				disable=not logging.is_progress_bar_enabled()       ,				desc='''Creating SQL from Arrow format'''       ,				):
																			written += self._batch_sql((offset, index, to_sql_kwargs)					)
									else:
														snake_case_			,snake_case_									=				len(self.dataset					), self.batch_size
														with multiprocessing.Pool(self.num_proc					) as pool:
																			for num_rows in logging.tqdm(
																			    pool.imap(
																			        self._batch_sql       ,				[(offset, index, to_sql_kwargs) for offset in range(0       ,				lowercase_       ,				lowercase_					)]       ,				)       ,				total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size       ,				unit='''ba'''       ,				disable=not logging.is_progress_bar_enabled()       ,				desc='''Creating SQL from Arrow format'''       ,				):
																								written += num_rows

									return written






'''simple docstring'''


_lowerCamelCase	:       int      	=	"0.21.0"

from .accelerator import Accelerator
from .big_modeling import (
    cpu_offload,
    cpu_offload_with_hook,
    disk_offload,
    dispatch_model,
    init_empty_weights,
    init_on_device,
    load_checkpoint_and_dispatch,
)
from .data_loader import skip_first_batches
from .launchers import debug_launcher, notebook_launcher
from .state import PartialState
from .utils import (
    DeepSpeedPlugin,
    DistributedDataParallelKwargs,
    DistributedType,
    FullyShardedDataParallelPlugin,
    GradScalerKwargs,
    InitProcessGroupKwargs,
    find_executable_batch_size,
    infer_auto_device_map,
    is_rich_available,
    load_checkpoint_in_model,
    synchronize_rng_states,
)


if is_rich_available():
   from .utils import rich

