


"""simple docstring"""




import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


lowercase_			     =  datasets.utils.logging.get_logger(__name__)
@dataclass
class 					__lowerCAmelCase					(   datasets.BuilderConfig       ):


		'''simple docstring'''



		__UpperCAmelCase		:			Optional[datasets.Features]										=				None
		__UpperCAmelCase		:			str										=				"utf-8"
		__UpperCAmelCase		:			Optional[str]										=				None
		__UpperCAmelCase		:			Optional[str]										=				None
		__UpperCAmelCase		:			bool										=				True  # deprecated
		__UpperCAmelCase		:			Optional[int]										=				None  # deprecated
		__UpperCAmelCase		:			int										=				1_0 << 2_0  # 10MB
		__UpperCAmelCase		:			Optional[bool]										=				None







class 					__lowerCAmelCase					(   datasets.ArrowBasedBuilder       ):


		'''simple docstring'''



		__UpperCAmelCase		:			Optional[Any]										=				JsonConfig

		def 			__UpperCAmelCase       (  self      ):
				if self.config.block_size is not None:
						logger.warning('''The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'''      )
						__a           =			self.config.block_size
				if self.config.use_threads is not True:
						logger.warning(
						    '''The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'''      )
				if self.config.newlines_in_values is not None:
						raise ValueError('''The JSON loader parameter `newlines_in_values` is no longer supported'''      )
				return datasets.DatasetInfo(features=self.config.features      )

		def 			__UpperCAmelCase       (  self							, _a      ):
				if not self.config.data_files:
						raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}'''      )
				__a           =			dl_manager.download_and_extract(self.config.data_files      )
				if isinstance(_a							, (str, list, tuple)      ):
						__a           =			data_files
						if isinstance(_a							, _a      ):
								__a           =			[files]
						__a           =			[dl_manager.iter_files(_a      ) for file in files]
						return [datasets.SplitGenerator(name=datasets.Split.TRAIN							, gen_kwargs={'''files''': files}      )]
				__a           =			[]
				for split_name, files in data_files.items():
						if isinstance(_a							, _a      ):
								__a           =			[files]
						__a           =			[dl_manager.iter_files(_a      ) for file in files]
						splits.append(datasets.SplitGenerator(name=_a							, gen_kwargs={'''files''': files}      )      )
				return splits

		def 			__UpperCAmelCase       (  self							, _a      ):
				if self.config.features is not None:
						# adding missing columns
						for column_name in set(self.config.features      ) - set(pa_table.column_names      ):
								__a           =			self.config.features.arrow_schema.field(_a      ).type
								__a           =			pa_table.append_column(_a							, pa.array([None] * len(_a      )							, type=_a      )      )
						# more expensive cast to support nested structures with keys in a different order
						# allows str <-> int/float or str to Audio for example
						__a           =			table_cast(_a							, self.config.features.arrow_schema      )
				return pa_table





		def 			__UpperCAmelCase       (  self							, _a      ):
				for file_idx, file in enumerate(itertools.chain.from_iterable(_a      )      ):
						# If the file is one json object and if we need to look at the list of items in one specific field
						if self.config.field is not None:
								with open(_a							, encoding=self.config.encoding							, errors=self.config.encoding_errors      ) as f:
										__a           =			json.load(_a      )

								# We keep only the field we are interested in
								__a           =			dataset[self.config.field]

								# We accept two format: a list of dicts or a dict of lists
								if isinstance(_a							, (list, tuple)      ):
										__a           =			set().union(*[row.keys() for row in dataset]      )
										__a           =			{col: [row.get(_a      ) for row in dataset] for col in keys}
								else:
										__a           =			dataset
								__a           =			pa.Table.from_pydict(_a      )
								yield file_idx, self._cast_table(_a      )

						# If the file has one json object per line
						else:
								with open(_a							, '''rb'''      ) as f:
										__a           =			0
										# Use block_size equal to the chunk size divided by 32 to leverage multithreading
										# Set a default minimum value of 16kB if the chunk size is really small
										__a           =			max(self.config.chunksize // 32							, 16 << 10      )
										__a           =			(
										    self.config.encoding_errors if self.config.encoding_errors is not None else '''strict'''
										)
										while True:
												__a           =			f.read(self.config.chunksize      )
												if not batch:
														break
												# Finish current line
												try:
														batch += f.readline()
												except (AttributeError, io.UnsupportedOperation):
														batch += readline(_a      )
												# PyArrow only accepts utf-8 encoded bytes
												if self.config.encoding != "utf-8":
														__a           =			batch.decode(self.config.encoding							, errors=_a      ).encode('''utf-8'''      )
												try:
														while True:
																try:
																		__a           =			paj.read_json(
																		    io.BytesIO(_a      )							, read_options=paj.ReadOptions(block_size=_a      )      )
																		break
																except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																		if (
																		    isinstance(_a							, pa.ArrowInvalid      )
																		    and "straddling" not in str(_a      )
																		    or block_size > len(_a      )
																		):
																				raise
																		else:
																				# Increase the block size in case it was too small.
																				# The block size will be reset for the next file.
																				logger.debug(
																				    f'''Batch of {len(_a      )} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'''      )
																				block_size *= 2
												except pa.ArrowInvalid as e:
														try:
																with open(
																    _a							, encoding=self.config.encoding							, errors=self.config.encoding_errors      ) as f:
																		__a           =			json.load(_a      )
														except json.JSONDecodeError:
																logger.error(f'''Failed to read file \'{file}\' with error {type(_a      )}: {e}'''      )
																raise e
														# If possible, parse the file as a list of json objects and exit the loop
														if isinstance(_a							, _a      ):  # list is the only sequence type supported in JSON
																try:
																		__a           =			set().union(*[row.keys() for row in dataset]      )
																		__a           =			{col: [row.get(_a      ) for row in dataset] for col in keys}
																		__a           =			pa.Table.from_pydict(_a      )
																except (pa.ArrowInvalid, AttributeError) as e:
																		logger.error(f'''Failed to read file \'{file}\' with error {type(_a      )}: {e}'''      )
																		raise ValueError(f'''Not able to read records in the JSON file at {file}.'''      ) from None
																yield file_idx, self._cast_table(_a      )
																break
														else:
																logger.error(f'''Failed to read file \'{file}\' with error {type(_a      )}: {e}'''      )
																raise ValueError(
																    f'''Not able to read records in the JSON file at {file}. '''
																    f'''You should probably indicate the field of the JSON file containing your records. '''
																    f'''This JSON file contain the following fields: {str(list(dataset.keys()      )      )}. '''
																    f'''Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '''      ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
												yield (file_idx, batch_idx), self._cast_table(_a      )
												batch_idx += 1










"""simple docstring"""




import warnings

from ..trainer import Trainer
from ..utils import logging


lowercase_			     =  logging.get_logger(__name__)
class 					__lowerCAmelCase					(   __SCREAMING_SNAKE_CASE       ):


		'''simple docstring'''



		def __init__(  self							, _a=None							, **_a      ):
				warnings.warn(
				    '''`SageMakerTrainer` is deprecated and will be removed in v5 of Transformers. You can use `Trainer` '''
				    '''instead.'''							, _a							, )
				super().__init__(args=_a							, **_a      )






