


import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


_snake_case     :		Union[str, Any]						=				datasets.utils.logging.get_logger(__name__)

@dataclass
class 				a		(datasets.BuilderConfig      ):




							"""simple docstring"""


							__UpperCAmelCase    :							Optional[datasets.Features]       =  None
							__UpperCAmelCase    :							str       =  "utf-8"
							__UpperCAmelCase    :							Optional[str]       =  None
							__UpperCAmelCase    :							Optional[str]       =  None
							__UpperCAmelCase    :							bool       =  True  # deprecated
							__UpperCAmelCase    :							Optional[int]       =  None  # deprecated
							__UpperCAmelCase    :							int       =  10 << 20  # 10MB
							__UpperCAmelCase    :							Optional[bool]       =  None

class 				a		(datasets.ArrowBasedBuilder      ):




							"""simple docstring"""


							__UpperCAmelCase    :							str       =  JsonConfig

							def     __snake_case					(							self       :  List[str]				)	->    List[str]:
														if self.config.block_size is not None:
																					logger.warning("The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"				)
																					__snake_case		:			Optional[int]							 =   self.config.block_size
														if self.config.use_threads is not True:
																					logger.warning(
																					    "The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."				)
														if self.config.newlines_in_values is not None:
																					raise ValueError("The JSON loader parameter `newlines_in_values` is no longer supported"				)
														return datasets.DatasetInfo(features=self.config.features				)

							def     __snake_case					(							self       :  Dict    ,	lowerCamelCase       :  Optional[int]				)	->    Dict:
														if not self.config.data_files:
																					raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}'				)
														__snake_case		:			int							 =   dl_manager.download_and_extract(self.config.data_files				)
														if isinstance(__a    ,	(str, list, tuple)				):
																					__snake_case		:			List[Any]							 =   data_files
																					if isinstance(__a    ,	__a				):
																												__snake_case		:			int							 =   [files]
																					__snake_case		:			Optional[int]							 =   [dl_manager.iter_files(__a				) for file in files]
																					return [datasets.SplitGenerator(name=datasets.Split.TRAIN    ,	gen_kwargs={"files": files}				)]
														__snake_case		:			Optional[int]							 =   []
														for split_name, files in data_files.items():
																					if isinstance(__a    ,	__a				):
																												__snake_case		:			Union[str, Any]							 =   [files]
																					__snake_case		:			Optional[Any]							 =   [dl_manager.iter_files(__a				) for file in files]
																					splits.append(datasets.SplitGenerator(name=__a    ,	gen_kwargs={"files": files}				)				)
														return splits

							def     __snake_case					(							self       :  Optional[int]    ,	lowerCamelCase       :  Any				)	->    Union[str, Any]:
														if self.config.features is not None:
																					# adding missing columns
																					for column_name in set(self.config.features				) - set(pa_table.column_names				):
																												__snake_case		:			Tuple							 =   self.config.features.arrow_schema.field(__a				).type
																												__snake_case		:			Tuple							 =   pa_table.append_column(__a    ,	pa.array([None] * len(__a				)    ,	type=__a				)				)
																					# more expensive cast to support nested structures with keys in a different order
																					# allows str <-> int/float or str to Audio for example
																					__snake_case		:			int							 =   table_cast(__a    ,	self.config.features.arrow_schema				)
														return pa_table






							def     __snake_case					(							self       :  Optional[Any]    ,	lowerCamelCase       :  Optional[int]				)	->    Dict:
														for file_idx, file in enumerate(itertools.chain.from_iterable(__a				)				):
																					# If the file is one json object and if we need to look at the list of items in one specific field
																					if self.config.field is not None:
																												with open(__a    ,	encoding=self.config.encoding    ,	errors=self.config.encoding_errors				) as f:
																																			__snake_case		:			Any							 =   json.load(__a				)

																												# We keep only the field we are interested in
																												__snake_case		:			Tuple							 =   dataset[self.config.field]

																												# We accept two format: a list of dicts or a dict of lists
																												if isinstance(__a    ,	(list, tuple)				):
																																			__snake_case		:			Any							 =   set().union(*[row.keys() for row in dataset]				)
																																			__snake_case		:			Union[str, Any]							 =   {col: [row.get(__a				) for row in dataset] for col in keys}
																												else:
																																			__snake_case		:			Optional[Any]							 =   dataset
																												__snake_case		:			List[Any]							 =   pa.Table.from_pydict(__a				)
																												yield file_idx, self._cast_table(__a				)

																					# If the file has one json object per line
																					else:
																												with open(__a    ,	"rb"				) as f:
																																			__snake_case		:			Union[str, Any]							 =   0
																																			# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																																			# Set a default minimum value of 16kB if the chunk size is really small
																																			__snake_case		:			Optional[int]							 =   max(self.config.chunksize // 32    ,	16 << 10				)
																																			__snake_case		:			int							 =   (
																																			    self.config.encoding_errors if self.config.encoding_errors is not None else "strict"
																																			)
																																			while True:
																																										__snake_case		:			List[str]							 =   f.read(self.config.chunksize				)
																																										if not batch:
																																																	break
																																										# Finish current line
																																										try:
																																																	batch += f.readline()
																																										except (AttributeError, io.UnsupportedOperation):
																																																	batch += readline(__a				)
																																										# PyArrow only accepts utf-8 encoded bytes
																																										if self.config.encoding != "utf-8":
																																																	__snake_case		:			List[str]							 =   batch.decode(self.config.encoding    ,	errors=__a				).encode("utf-8"				)
																																										try:
																																																	while True:
																																																								try:
																																																															__snake_case		:			List[Any]							 =   paj.read_json(
																																																															    io.BytesIO(__a				)    ,	read_options=paj.ReadOptions(block_size=__a				)				)
																																																															break
																																																								except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																																																															if (
																																																															    isinstance(__a    ,	pa.ArrowInvalid				)
																																																															    and "straddling" not in str(__a				)
																																																															    or block_size > len(__a				)
																																																															):
																																																																						raise
																																																															else:
																																																																						# Increase the block size in case it was too small.
																																																																						# The block size will be reset for the next file.
																																																																						logger.debug(
																																																																						    F'Batch of {len(__a				)} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'				)
																																																																						block_size *= 2
																																										except pa.ArrowInvalid as e:
																																																	try:
																																																								with open(
																																																								    __a    ,	encoding=self.config.encoding    ,	errors=self.config.encoding_errors				) as f:
																																																															__snake_case		:			Dict							 =   json.load(__a				)
																																																	except json.JSONDecodeError:
																																																								logger.error(F'Failed to read file \'{file}\' with error {type(__a				)}: {e}'				)
																																																								raise e
																																																	# If possible, parse the file as a list of json objects and exit the loop
																																																	if isinstance(__a    ,	__a				):  # list is the only sequence type supported in JSON
																																																								try:
																																																															__snake_case		:			Dict							 =   set().union(*[row.keys() for row in dataset]				)
																																																															__snake_case		:			int							 =   {col: [row.get(__a				) for row in dataset] for col in keys}
																																																															__snake_case		:			Any							 =   pa.Table.from_pydict(__a				)
																																																								except (pa.ArrowInvalid, AttributeError) as e:
																																																															logger.error(F'Failed to read file \'{file}\' with error {type(__a				)}: {e}'				)
																																																															raise ValueError(F'Not able to read records in the JSON file at {file}.'				) from None
																																																								yield file_idx, self._cast_table(__a				)
																																																								break
																																																	else:
																																																								logger.error(F'Failed to read file \'{file}\' with error {type(__a				)}: {e}'				)
																																																								raise ValueError(
																																																								    F'Not able to read records in the JSON file at {file}. '
																																																								    F'You should probably indicate the field of the JSON file containing your records. '
																																																								    F'This JSON file contain the following fields: {str(list(dataset.keys()				)				)}. '
																																																								    F'Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '				) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																										yield (file_idx, batch_idx), self._cast_table(__a				)
																																										batch_idx += 1





"""simple docstring"""


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


A     :   Optional[int]			    =			{
    "configuration_altclip": [
        "ALTCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "AltCLIPConfig",
        "AltCLIPTextConfig",
        "AltCLIPVisionConfig",
    ],
    "processing_altclip": ["AltCLIPProcessor"],
}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	A     :   Tuple			    =			[
	    "ALTCLIP_PRETRAINED_MODEL_ARCHIVE_LIST",
	    "AltCLIPPreTrainedModel",
	    "AltCLIPModel",
	    "AltCLIPTextModel",
	    "AltCLIPVisionModel",
	]


if TYPE_CHECKING:
	from .configuration_altclip import (
	    ALTCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
	    AltCLIPConfig,
	    AltCLIPTextConfig,
	    AltCLIPVisionConfig,
	)
	from .processing_altclip import AltCLIPProcessor

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_altclip import (
		    ALTCLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
		    AltCLIPModel,
		    AltCLIPPreTrainedModel,
		    AltCLIPTextModel,
		    AltCLIPVisionModel,
		)


else:
	import sys

	A     :   int			    =			_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)




