'''simple docstring'''


import os
import pickle
import unittest
from collections import UserDict, namedtuple

import torch

from accelerate.test_utils.testing import require_cuda, require_torch_min_version
from accelerate.test_utils.training import RegressionModel
from accelerate.utils import (
    convert_outputs_to_fpaa,
    extract_model_from_parallel,
    find_device,
    listify,
    patch_environment,
    recursively_apply,
    send_to_device,
)


snake_case__	:     Optional[int]      =    namedtuple('''ExampleNamedTuple''', '''a b c''')






class 	__SCREAMING_SNAKE_CASE		(			unittest.TestCase			):







  '''simple docstring'''






  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  List[Any]  = torch.randn(5 ,						2		)
       UpperCAmelCase_				:  Optional[Any]  = torch.device('cuda'		) if torch.cuda.is_available() else torch.device('cpu'		)

       UpperCAmelCase_				:  Dict  = send_to_device(snake_case_ ,						snake_case_		)
       self.assertTrue(torch.equal(resulta.cpu() ,						snake_case_		)		)

       UpperCAmelCase_				:  Tuple  = send_to_device((tensor, [tensor, tensor], 1) ,						snake_case_		)
       self.assertIsInstance(snake_case_ ,						snake_case_		)
       self.assertTrue(torch.equal(resulta[0].cpu() ,						snake_case_		)		)
       self.assertIsInstance(resulta[1] ,						snake_case_		)
       self.assertTrue(torch.equal(resulta[1][0].cpu() ,						snake_case_		)		)
       self.assertTrue(torch.equal(resulta[1][1].cpu() ,						snake_case_		)		)
       self.assertEqual(resulta[2] ,						1		)

       UpperCAmelCase_				:  List[Any]  = send_to_device({'a': tensor, 'b': [tensor, tensor], 'c': 1} ,						snake_case_		)
       self.assertIsInstance(snake_case_ ,						snake_case_		)
       self.assertTrue(torch.equal(resulta['a'].cpu() ,						snake_case_		)		)
       self.assertIsInstance(resulta['b'] ,						snake_case_		)
       self.assertTrue(torch.equal(resulta['b'][0].cpu() ,						snake_case_		)		)
       self.assertTrue(torch.equal(resulta['b'][1].cpu() ,						snake_case_		)		)
       self.assertEqual(resulta['c'] ,						1		)

       UpperCAmelCase_				:  Tuple  = send_to_device(ExampleNamedTuple(a=snake_case_ ,						b=[tensor, tensor] ,						c=1		) ,						snake_case_		)
       self.assertIsInstance(snake_case_ ,						snake_case_		)
       self.assertTrue(torch.equal(resulta.a.cpu() ,						snake_case_		)		)
       self.assertIsInstance(resulta.b ,						snake_case_		)
       self.assertTrue(torch.equal(resulta.b[0].cpu() ,						snake_case_		)		)
       self.assertTrue(torch.equal(resulta.b[1].cpu() ,						snake_case_		)		)
       self.assertEqual(resulta.c ,						1		)

       UpperCAmelCase_				:  int  = send_to_device(UserDict({'a': tensor, 'b': [tensor, tensor], 'c': 1}		) ,						snake_case_		)
       self.assertIsInstance(snake_case_ ,						snake_case_		)
       self.assertTrue(torch.equal(resulta['a'].cpu() ,						snake_case_		)		)
       self.assertIsInstance(resulta['b'] ,						snake_case_		)
       self.assertTrue(torch.equal(resulta['b'][0].cpu() ,						snake_case_		)		)
       self.assertTrue(torch.equal(resulta['b'][1].cpu() ,						snake_case_		)		)
       self.assertEqual(resulta['c'] ,						1		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       with self.assertRaises(snake_case_		) as cm:
            UpperCAmelCase_				:  Optional[Any]  = recursively_apply(torch.tensor ,						(torch.tensor(1		), 1) ,						error_on_other_type=snake_case_		)
       self.assertEqual(
           str(cm.exception		) ,						'Unsupported types (<class \'int\'>) passed to `tensor`. Only nested list/tuple/dicts of objects that are valid for `is_torch_tensor` should be passed.' ,						)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Dict  = torch.tensor([1, 2, 3, 4, 5]		)
       self.assertEqual(listify(snake_case_		) ,						[1, 2, 3, 4, 5]		)

       UpperCAmelCase_				:  List[str]  = torch.tensor([[1, 2, 3, 4, 5], [6, 7, 8, 9, 1_0]]		)
       self.assertEqual(listify(snake_case_		) ,						[[1, 2, 3, 4, 5], [6, 7, 8, 9, 1_0]]		)

       UpperCAmelCase_				:  int  = torch.tensor([[[1, 2, 3, 4, 5], [6, 7, 8, 9, 1_0]], [[1_1, 1_2, 1_3, 1_4, 1_5], [1_6, 1_7, 1_8, 1_9, 2_0]]]		)
       self.assertEqual(
           listify(snake_case_		) ,						[[[1, 2, 3, 4, 5], [6, 7, 8, 9, 1_0]], [[1_1, 1_2, 1_3, 1_4, 1_5], [1_6, 1_7, 1_8, 1_9, 2_0]]]		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       with patch_environment(aa=1 ,						BB=2		):
            self.assertEqual(os.environ.get('AA'		) ,						'1'		)
            self.assertEqual(os.environ.get('BB'		) ,						'2'		)

       self.assertNotIn('AA' ,						os.environ		)
       self.assertNotIn('BB' ,						os.environ		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Optional[int]  = RegressionModel()
       UpperCAmelCase_				:  Optional[Any]  = model.forward
       UpperCAmelCase_				:  List[Any]  = convert_outputs_to_fpaa(model.forward		)
       UpperCAmelCase_				:  Any  = extract_model_from_parallel(snake_case_ ,						keep_fpaa_wrapper=snake_case_		)
       UpperCAmelCase_				:  Optional[Any]  = pickle.dumps(snake_case_		)


  @require_cuda
  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Tuple  = RegressionModel()
       UpperCAmelCase_				:  Tuple  = model.forward
       UpperCAmelCase_				:  Optional[Any]  = torch.cuda.amp.autocast(dtype=torch.floataa		)(model.forward		)
       UpperCAmelCase_				:  int  = convert_outputs_to_fpaa(model.forward		)
       UpperCAmelCase_				:  Optional[Any]  = extract_model_from_parallel(snake_case_ ,						keep_fpaa_wrapper=snake_case_		)
       UpperCAmelCase_				:  List[Any]  = pickle.dumps(snake_case_		)


  @require_cuda
  @require_torch_min_version(version='2.0'		)
  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Tuple  = RegressionModel()
       UpperCAmelCase_				:  str  = model.forward
       UpperCAmelCase_				:  str  = torch.cuda.amp.autocast(dtype=torch.floataa		)(model.forward		)
       UpperCAmelCase_				:  List[str]  = convert_outputs_to_fpaa(model.forward		)
       UpperCAmelCase_				:  List[Any]  = torch.compile(model.forward ,						backend='inductor'		)
       UpperCAmelCase_				:  List[Any]  = torch.randn(4 ,						1_0		).cuda()
       UpperCAmelCase_				:  List[str]  = model(snake_case_		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  List[Any]  = RegressionModel()
       # could also do a test with DistributedDataParallel, but difficult to run on CPU or single GPU
       UpperCAmelCase_				:  List[Any]  = torch.nn.parallel.DataParallel(snake_case_		)
       UpperCAmelCase_				:  List[str]  = extract_model_from_parallel(snake_case_		)

       self.assertEqual(snake_case_ ,						snake_case_		)


  @require_torch_min_version(version='2.0'		)
  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Optional[Any]  = RegressionModel()
       UpperCAmelCase_				:  List[str]  = torch.compile(snake_case_		)

       # could also do a test with DistributedDataParallel, but difficult to run on CPU or single GPU
       UpperCAmelCase_				:  Optional[int]  = torch.nn.parallel.DataParallel(snake_case_		)
       UpperCAmelCase_				:  List[str]  = torch.compile(snake_case_		)
       UpperCAmelCase_				:  Union[str, Any]  = extract_model_from_parallel(snake_case_		)

       self.assertEqual(compiled_model._orig_mod ,						compiled_model_unwrapped._orig_mod		)






  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       self.assertEqual(find_device([1, 'a', torch.tensor([1, 2, 3]		)]		) ,						torch.device('cpu'		)		)
       self.assertEqual(find_device({'a': 1, 'b': torch.tensor([1, 2, 3]		)}		) ,						torch.device('cpu'		)		)
       self.assertIsNone(find_device([1, 'a']		)		)







'''simple docstring'''


import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


if TYPE_CHECKING:
 import sqlitea

 import sqlalchemy


snake_case__	:     int      =    datasets.utils.logging.get_logger(__name__)






@dataclass
class 	__SCREAMING_SNAKE_CASE		(			datasets.BuilderConfig			):







  '''simple docstring'''






  lowerCamelCase_						:Union[str, "sqlalchemy.sql.Selectable"]		       =						None
  lowerCamelCase_						:Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"]		       =						None
  lowerCamelCase_						:Optional[Union[str, List[str]]]		       =						None
  lowerCamelCase_						:bool		       =						True
  lowerCamelCase_						:Optional[Union[List, Tuple, Dict]]		       =						None
  lowerCamelCase_						:Optional[Union[List, Dict]]		       =						None
  lowerCamelCase_						:Optional[List[str]]		       =						None
  lowerCamelCase_						:Optional[int]		       =						1_0000
  lowerCamelCase_						:Optional[datasets.Features]		       =						None


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       if self.sql is None:
            raise ValueError('sql must be specified'		)
       if self.con is None:
            raise ValueError('con must be specified'		)


  def _UpperCamelCase	(					self ,						snake_case_ ,						snake_case_ = None ,						):




       '''simple docstring'''

       UpperCAmelCase_				:  List[Any]  = config_kwargs.copy()
       # We need to stringify the Selectable object to make its hash deterministic

       # The process of stringifying is explained here: http://docs.sqlalchemy.org/en/latest/faq/sqlexpressions.html
       UpperCAmelCase_				:  str  = config_kwargs['sql']
       if not isinstance(snake_case_ ,						snake_case_		):
            if datasets.config.SQLALCHEMY_AVAILABLE and "sqlalchemy" in sys.modules:
                 import sqlalchemy

                 if isinstance(snake_case_ ,						sqlalchemy.sql.Selectable		):
                      UpperCAmelCase_				:  Any  = sqlalchemy.create_engine(config_kwargs['con'].split('://'		)[0] + '://'		)
                      UpperCAmelCase_				:  int  = str(sql.compile(dialect=engine.dialect		)		)
                      UpperCAmelCase_				:  str  = sql_str
                 else:
                      raise TypeError(
                          F'''Supported types for \'sql\' are string and sqlalchemy.sql.Selectable but got {type(snake_case_		)}: {sql}'''		)
            else:
                 raise TypeError(
                     F'''Supported types for \'sql\' are string and sqlalchemy.sql.Selectable but got {type(snake_case_		)}: {sql}'''		)
       UpperCAmelCase_				:  Union[str, Any]  = config_kwargs['con']
       if not isinstance(snake_case_ ,						snake_case_		):
            UpperCAmelCase_				:  Optional[Any]  = id(snake_case_		)
            logger.info(
                F'''SQL connection \'con\' of type {type(snake_case_		)} couldn\'t be hashed properly. To enable hashing, specify \'con\' as URI string instead.'''		)

       return super().create_config_id(snake_case_ ,						custom_features=snake_case_		)






  @property
  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  List[Any]  = {
           'index_col': self.index_col,
           'columns': self.columns,
           'params': self.params,
           'coerce_float': self.coerce_float,
           'parse_dates': self.parse_dates,
       }
       return pd_read_sql_kwargs







class 	__SCREAMING_SNAKE_CASE		(			datasets.ArrowBasedBuilder			):







  '''simple docstring'''






  lowerCamelCase_						:Any		       =						SqlConfig


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       return datasets.DatasetInfo(features=self.config.features		)


  def _UpperCamelCase	(					self ,						snake_case_		):




       '''simple docstring'''

       return [datasets.SplitGenerator(name=datasets.Split.TRAIN ,						gen_kwargs={}		)]


  def _UpperCamelCase	(					self ,						snake_case_		):




       '''simple docstring'''

       if self.config.features is not None:
            UpperCAmelCase_				:  Tuple  = self.config.features.arrow_schema
            if all(not require_storage_cast(snake_case_		) for feature in self.config.features.values()		):
                 # cheaper cast
                 UpperCAmelCase_				:  Tuple  = pa.Table.from_arrays([pa_table[field.name] for field in schema] ,						schema=snake_case_		)
            else:
                 # more expensive cast; allows str <-> int/float or str to Audio for example
                 UpperCAmelCase_				:  Union[str, Any]  = table_cast(snake_case_ ,						snake_case_		)
       return pa_table






  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Optional[Any]  = self.config.chunksize
       UpperCAmelCase_				:  Optional[Any]  = pd.read_sql(
           self.config.sql ,						self.config.con ,						chunksize=snake_case_ ,						**self.config.pd_read_sql_kwargs		)
       UpperCAmelCase_				:  int  = [sql_reader] if chunksize is None else sql_reader
       for chunk_idx, df in enumerate(snake_case_		):
            UpperCAmelCase_				:  Optional[Any]  = pa.Table.from_pandas(snake_case_		)
            yield chunk_idx, self._cast_table(snake_case_		)






