
"""simple docstring"""


import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


_lowercase							      = datasets.utils.logging.get_logger(__name__)







@dataclass
class  lowerCAmelCase_ (					datasets.BuilderConfig			):



						'''simple docstring'''


						_lowerCamelCase:			Optional[datasets.Features]					=			None
						_lowerCamelCase:			str					=			"utf-8"
						_lowerCamelCase:			Optional[str]					=			None
						_lowerCamelCase:			Optional[str]					=			None
						_lowerCamelCase:			bool					=			True  # deprecated
						_lowerCamelCase:			Optional[int]					=			None  # deprecated
						_lowerCamelCase:			int					=			10 << 20  # 10MB
						_lowerCamelCase:			Optional[bool]					=			None







class  lowerCAmelCase_ (					datasets.ArrowBasedBuilder			):



						'''simple docstring'''


						_lowerCamelCase:			Optional[int]					=			JsonConfig


						def   _SCREAMING_SNAKE_CASE					(       self   :					Optional[int]					)   ->      List[Any]:
							if self.config.block_size is not None:
								logger.warning('The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'					)
								A        =      self.config.block_size
							if self.config.use_threads is not True:
								logger.warning(
								    'The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'					)
							if self.config.newlines_in_values is not None:
								raise ValueError('The JSON loader parameter `newlines_in_values` is no longer supported'					)
							return datasets.DatasetInfo(features=self.config.features					)


						def   _SCREAMING_SNAKE_CASE					(       self   :					Any     ,A_   :					List[str]					)   ->      List[Any]:
							if not self.config.data_files:
								raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}'					)
							A        =      dl_manager.download_and_extract(self.config.data_files					)
							if isinstance(A_     ,(str, list, tuple)					):
								A        =      data_files
								if isinstance(A_     ,A_					):
									A        =      [files]
								A        =      [dl_manager.iter_files(A_					) for file in files]
								return [datasets.SplitGenerator(name=datasets.Split.TRAIN     ,gen_kwargs={'files': files}					)]
							A        =      []
							for split_name, files in data_files.items():
								if isinstance(A_     ,A_					):
									A        =      [files]
								A        =      [dl_manager.iter_files(A_					) for file in files]
								splits.append(datasets.SplitGenerator(name=A_     ,gen_kwargs={'files': files}					)					)
							return splits


						def   _SCREAMING_SNAKE_CASE					(       self   :					str     ,A_   :					pa.Table					)   ->      pa.Table:
							if self.config.features is not None:
								# adding missing columns
								for column_name in set(self.config.features					) - set(pa_table.column_names					):
									A        =      self.config.features.arrow_schema.field(A_					).type
									A        =      pa_table.append_column(A_     ,pa.array([None] * len(A_					)     ,type=A_					)					)
								# more expensive cast to support nested structures with keys in a different order
								# allows str <-> int/float or str to Audio for example
								A        =      table_cast(A_     ,self.config.features.arrow_schema					)
							return pa_table




						def   _SCREAMING_SNAKE_CASE					(       self   :					Any     ,A_   :					Tuple					)   ->      List[str]:
							for file_idx, file in enumerate(itertools.chain.from_iterable(A_					)					):
								# If the file is one json object and if we need to look at the list of items in one specific field
								if self.config.field is not None:
									with open(A_     ,encoding=self.config.encoding     ,errors=self.config.encoding_errors					) as f:
										A        =      json.load(A_					)

									# We keep only the field we are interested in
									A        =      dataset[self.config.field]

									# We accept two format: a list of dicts or a dict of lists
									if isinstance(A_     ,(list, tuple)					):
										A        =      set().union(*[row.keys() for row in dataset]					)
										A        =      {col: [row.get(A_					) for row in dataset] for col in keys}
									else:
										A        =      dataset
									A        =      pa.Table.from_pydict(A_					)
									yield file_idx, self._cast_table(A_					)

								# If the file has one json object per line
								else:
									with open(A_     ,'rb'					) as f:
										A        =      0
										# Use block_size equal to the chunk size divided by 32 to leverage multithreading
										# Set a default minimum value of 16kB if the chunk size is really small
										A        =      max(self.config.chunksize // 32     ,16 << 10					)
										A        =      (
										    self.config.encoding_errors if self.config.encoding_errors is not None else 'strict'
										)
										while True:
											A        =      f.read(self.config.chunksize					)
											if not batch:
												break
											# Finish current line
											try:
												batch += f.readline()
											except (AttributeError, io.UnsupportedOperation):
												batch += readline(A_					)
											# PyArrow only accepts utf-8 encoded bytes
											if self.config.encoding != "utf-8":
												A        =      batch.decode(self.config.encoding     ,errors=A_					).encode('utf-8'					)
											try:
												while True:
													try:
														A        =      paj.read_json(
														    io.BytesIO(A_					)     ,read_options=paj.ReadOptions(block_size=A_					)					)
														break
													except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
														if (
														    isinstance(A_     ,pa.ArrowInvalid					)
														    and "straddling" not in str(A_					)
														    or block_size > len(A_					)
														):
															raise
														else:
															# Increase the block size in case it was too small.
															# The block size will be reset for the next file.
															logger.debug(
															    F'Batch of {len(A_					)} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'					)
															block_size *= 2
											except pa.ArrowInvalid as e:
												try:
													with open(
													    A_     ,encoding=self.config.encoding     ,errors=self.config.encoding_errors					) as f:
														A        =      json.load(A_					)
												except json.JSONDecodeError:
													logger.error(F'Failed to read file \'{file}\' with error {type(A_					)}: {e}'					)
													raise e
												# If possible, parse the file as a list of json objects and exit the loop
												if isinstance(A_     ,A_					):  # list is the only sequence type supported in JSON
													try:
														A        =      set().union(*[row.keys() for row in dataset]					)
														A        =      {col: [row.get(A_					) for row in dataset] for col in keys}
														A        =      pa.Table.from_pydict(A_					)
													except (pa.ArrowInvalid, AttributeError) as e:
														logger.error(F'Failed to read file \'{file}\' with error {type(A_					)}: {e}'					)
														raise ValueError(F'Not able to read records in the JSON file at {file}.'					) from None
													yield file_idx, self._cast_table(A_					)
													break
												else:
													logger.error(F'Failed to read file \'{file}\' with error {type(A_					)}: {e}'					)
													raise ValueError(
													    F'Not able to read records in the JSON file at {file}. '
													    F'You should probably indicate the field of the JSON file containing your records. '
													    F'This JSON file contain the following fields: {str(list(dataset.keys()					)					)}. '
													    F'Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '					) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
											yield (file_idx, batch_idx), self._cast_table(A_					)
											batch_idx += 1







from ...utils import (
    OptionalDependencyNotAvailable,
    is_torch_available,
    is_transformers_available,
    is_transformers_version,
)


try:
	if not (is_transformers_available() and is_torch_available()):
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	from ...utils.dummy_torch_and_transformers_objects import ShapEPipeline
else:
	from .camera import create_pan_cameras
	from .pipeline_shap_e import ShapEPipeline
	from .pipeline_shap_e_img2img import ShapEImgaImgPipeline
	from .renderer import (
	    BoundingBoxVolume,
	    ImportanceRaySampler,
	    MLPNeRFModelOutput,
	    MLPNeRSTFModel,
	    ShapEParamsProjModel,
	    ShapERenderer,
	    StratifiedRaySampler,
	    VoidNeRFModel,
	)
