


'''simple docstring'''

import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


a_  :				Dict						= datasets.utils.logging.get_logger(__name__)





@dataclass
class      snake_case    (    datasets.BuilderConfig			):





  """simple docstring"""




  _lowerCamelCase		    =  None
  _lowerCamelCase		    =  "utf-8"
  _lowerCamelCase		    =  None
  _lowerCamelCase		    =  None
  _lowerCamelCase		    =  True  # deprecated
  _lowerCamelCase		    =  None  # deprecated
  _lowerCamelCase		    =  10 << 20  # 10MB
  _lowerCamelCase		    =  None





class      snake_case    (    datasets.ArrowBasedBuilder			):





  """simple docstring"""




  _lowerCamelCase		    =  JsonConfig






  def   snake_case		(    self     ):







   """simple docstring"""
   if self.config.block_size is not None:
    logger.warning("The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"     )
    lowerCamelCase_      =  self.config.block_size
   if self.config.use_threads is not True:
    logger.warning(
        "The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."     )
   if self.config.newlines_in_values is not None:
    raise ValueError("The JSON loader parameter `newlines_in_values` is no longer supported"     )
   return datasets.DatasetInfo(features=self.config.features     )






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   if not self.config.data_files:
    raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}'''     )
   lowerCamelCase_      =  dl_manager.download_and_extract(self.config.data_files     )
   if isinstance(UpperCamelCase			,  (str, list, tuple)     ):
    lowerCamelCase_      =  data_files
    if isinstance(UpperCamelCase			,  UpperCamelCase     ):
     lowerCamelCase_      =  [files]
    lowerCamelCase_      =  [dl_manager.iter_files(UpperCamelCase     ) for file in files]
    return [datasets.SplitGenerator(name=datasets.Split.TRAIN			,  gen_kwargs={"files": files}     )]
   lowerCamelCase_      =  []
   for split_name, files in data_files.items():
    if isinstance(UpperCamelCase			,  UpperCamelCase     ):
     lowerCamelCase_      =  [files]
    lowerCamelCase_      =  [dl_manager.iter_files(UpperCamelCase     ) for file in files]
    splits.append(datasets.SplitGenerator(name=UpperCamelCase			,  gen_kwargs={"files": files}     )     )
   return splits






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   if self.config.features is not None:
    # adding missing columns
    for column_name in set(self.config.features     ) - set(pa_table.column_names     ):
     lowerCamelCase_      =  self.config.features.arrow_schema.field(UpperCamelCase     ).type
     lowerCamelCase_      =  pa_table.append_column(UpperCamelCase			,  pa.array([None] * len(UpperCamelCase     )			,  type=UpperCamelCase     )     )
    # more expensive cast to support nested structures with keys in a different order
    # allows str <-> int/float or str to Audio for example
    lowerCamelCase_      =  table_cast(UpperCamelCase			,  self.config.features.arrow_schema     )
   return pa_table







  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   for file_idx, file in enumerate(itertools.chain.from_iterable(UpperCamelCase     )     ):
    # If the file is one json object and if we need to look at the list of items in one specific field
    if self.config.field is not None:
     with open(UpperCamelCase			,  encoding=self.config.encoding			,  errors=self.config.encoding_errors     ) as f:
      lowerCamelCase_      =  json.load(UpperCamelCase     )

     # We keep only the field we are interested in
     lowerCamelCase_      =  dataset[self.config.field]

     # We accept two format: a list of dicts or a dict of lists
     if isinstance(UpperCamelCase			,  (list, tuple)     ):
      lowerCamelCase_      =  set().union(*[row.keys() for row in dataset]     )
      lowerCamelCase_      =  {col: [row.get(UpperCamelCase     ) for row in dataset] for col in keys}
     else:
      lowerCamelCase_      =  dataset
     lowerCamelCase_      =  pa.Table.from_pydict(UpperCamelCase     )
     yield file_idx, self._cast_table(UpperCamelCase     )

    # If the file has one json object per line
    else:
     with open(UpperCamelCase			,  "rb"     ) as f:
      lowerCamelCase_      =  0
      # Use block_size equal to the chunk size divided by 32 to leverage multithreading
      # Set a default minimum value of 16kB if the chunk size is really small
      lowerCamelCase_      =  max(self.config.chunksize // 32			,  16 << 10     )
      lowerCamelCase_      =  (
          self.config.encoding_errors if self.config.encoding_errors is not None else "strict"
      )
      while True:
       lowerCamelCase_      =  f.read(self.config.chunksize     )
       if not batch:
        break
       # Finish current line
       try:
        batch += f.readline()
       except (AttributeError, io.UnsupportedOperation):
        batch += readline(UpperCamelCase     )
       # PyArrow only accepts utf-8 encoded bytes
       if self.config.encoding != "utf-8":
        lowerCamelCase_      =  batch.decode(self.config.encoding			,  errors=UpperCamelCase     ).encode("utf-8"     )
       try:
        while True:
         try:
          lowerCamelCase_      =  paj.read_json(
              io.BytesIO(UpperCamelCase     )			,  read_options=paj.ReadOptions(block_size=UpperCamelCase     )     )
          break
         except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
          if (
              isinstance(UpperCamelCase			,  pa.ArrowInvalid     )
              and "straddling" not in str(UpperCamelCase     )
              or block_size > len(UpperCamelCase     )
          ):
           raise
          else:
           # Increase the block size in case it was too small.
           # The block size will be reset for the next file.
           logger.debug(
               f'''Batch of {len(UpperCamelCase     )} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'''     )
           block_size *= 2
       except pa.ArrowInvalid as e:
        try:
         with open(
             UpperCamelCase			,  encoding=self.config.encoding			,  errors=self.config.encoding_errors     ) as f:
          lowerCamelCase_      =  json.load(UpperCamelCase     )
        except json.JSONDecodeError:
         logger.error(f'''Failed to read file \'{file}\' with error {type(UpperCamelCase     )}: {e}'''     )
         raise e
        # If possible, parse the file as a list of json objects and exit the loop
        if isinstance(UpperCamelCase			,  UpperCamelCase     ):  # list is the only sequence type supported in JSON
         try:
          lowerCamelCase_      =  set().union(*[row.keys() for row in dataset]     )
          lowerCamelCase_      =  {col: [row.get(UpperCamelCase     ) for row in dataset] for col in keys}
          lowerCamelCase_      =  pa.Table.from_pydict(UpperCamelCase     )
         except (pa.ArrowInvalid, AttributeError) as e:
          logger.error(f'''Failed to read file \'{file}\' with error {type(UpperCamelCase     )}: {e}'''     )
          raise ValueError(f'''Not able to read records in the JSON file at {file}.'''     ) from None
         yield file_idx, self._cast_table(UpperCamelCase     )
         break
        else:
         logger.error(f'''Failed to read file \'{file}\' with error {type(UpperCamelCase     )}: {e}'''     )
         raise ValueError(
             f'''Not able to read records in the JSON file at {file}. '''
             f'''You should probably indicate the field of the JSON file containing your records. '''
             f'''This JSON file contain the following fields: {str(list(dataset.keys()     )     )}. '''
             f'''Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '''     ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
       yield (file_idx, batch_idx), self._cast_table(UpperCamelCase     )
       batch_idx += 1







'''simple docstring'''

# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from argparse import ArgumentParser

from accelerate.commands.config import get_config_parser
from accelerate.commands.env import env_command_parser
from accelerate.commands.launch import launch_command_parser
from accelerate.commands.test import test_command_parser
from accelerate.commands.tpu import tpu_command_parser





def __snake_case					(      ):
 lowerCamelCase_      =  ArgumentParser("Accelerate CLI tool"					,				usage="accelerate <command> [<args>]"					,				allow_abbrev=UpperCAmelCase_		)
 lowerCamelCase_      =  parser.add_subparsers(help="accelerate command helpers"		)

 # Register commands
 get_config_parser(subparsers=UpperCAmelCase_		)
 env_command_parser(subparsers=UpperCAmelCase_		)
 launch_command_parser(subparsers=UpperCAmelCase_		)
 tpu_command_parser(subparsers=UpperCAmelCase_		)
 test_command_parser(subparsers=UpperCAmelCase_		)

 # Let's go
 lowerCamelCase_      =  parser.parse_args()

 if not hasattr(UpperCAmelCase_					,				"func"		):
  parser.print_help()
  exit(1		)

 # Run
 args.func(UpperCAmelCase_		)


if __name__ == "__main__":
     main()



