

import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


UpperCAmelCase__			:  List[str]					     =     datasets.utils.logging.get_logger(__name__)







@dataclass
class 							a__							( datasets.BuilderConfig							):







  """simple docstring"""




  UpperCAmelCase__   :			Optional[datasets.Features]      							=None
  UpperCAmelCase__   :			str      							="utf-8"
  UpperCAmelCase__   :			Optional[str]      							=None
  UpperCAmelCase__   :			int      							=1_0 << 2_0  # 10MB
  UpperCAmelCase__   :			bool      							=False
  UpperCAmelCase__   :			str      							="line"







class 							a__							( datasets.ArrowBasedBuilder							):







  """simple docstring"""




  UpperCAmelCase__   :			Dict      							=TextConfig


  def 			_lowercase   (      self     :			Tuple		)       ->int:
        """simple docstring"""







        return datasets.DatasetInfo(features=self.config.features		)


  def 			_lowercase   (      self     :			Tuple					,					UpperCAmelCase__     :			int		)       ->List[str]:
        """simple docstring"""







        if not self.config.data_files:
              raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}"		)
        SCREAMING_SNAKE_CASE			:   Optional[Any]	  = dl_manager.download_and_extract(self.config.data_files		)
        if isinstance(UpperCAmelCase__					,					(str, list, tuple)		):
              SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = data_files
              if isinstance(UpperCAmelCase__					,					UpperCAmelCase__		):
                    SCREAMING_SNAKE_CASE			:   Optional[int]	  = [files]
              SCREAMING_SNAKE_CASE			:   Dict	  = [dl_manager.iter_files(UpperCAmelCase__		) for file in files]
              return [datasets.SplitGenerator(name=datasets.Split.TRAIN					,					gen_kwargs={"""files""": files}		)]
        SCREAMING_SNAKE_CASE			:   List[Any]	  = []
        for split_name, files in data_files.items():
              if isinstance(UpperCAmelCase__					,					UpperCAmelCase__		):
                    SCREAMING_SNAKE_CASE			:   str	  = [files]
              SCREAMING_SNAKE_CASE			:   Any	  = [dl_manager.iter_files(UpperCAmelCase__		) for file in files]
              splits.append(datasets.SplitGenerator(name=UpperCAmelCase__					,					gen_kwargs={"""files""": files}		)		)
        return splits


  def 			_lowercase   (      self     :			Dict					,					UpperCAmelCase__     :			pa.Table		)       ->pa.Table:
        """simple docstring"""







        if self.config.features is not None:
              SCREAMING_SNAKE_CASE			:   List[Any]	  = self.config.features.arrow_schema
              if all(not require_storage_cast(UpperCAmelCase__		) for feature in self.config.features.values()		):
                    # cheaper cast
                    SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = pa_table.cast(UpperCAmelCase__		)
              else:
                    # more expensive cast; allows str <-> int/float or str to Audio for example
                    SCREAMING_SNAKE_CASE			:   List[Any]	  = table_cast(UpperCAmelCase__					,					UpperCAmelCase__		)
              return pa_table
        else:
              return pa_table.cast(pa.schema({"""text""": pa.string()}		)		)




  def 			_lowercase   (      self     :			Optional[int]					,					UpperCAmelCase__     :			str		)       ->str:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Optional[Any]	  = list(self.config.features		) if self.config.features is not None else ["""text"""]
        for file_idx, file in enumerate(itertools.chain.from_iterable(UpperCAmelCase__		)		):
              # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
              with open(UpperCAmelCase__					,					encoding=self.config.encoding					,					errors=self.config.errors		) as f:
                    if self.config.sample_by == "line":
                          SCREAMING_SNAKE_CASE			:   List[str]	  = 0
                          while True:
                                SCREAMING_SNAKE_CASE			:   List[str]	  = f.read(self.config.chunksize		)
                                if not batch:
                                      break
                                batch += f.readline()  # finish current line
                                # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
                                SCREAMING_SNAKE_CASE			:   Dict	  = StringIO(UpperCAmelCase__		).readlines()
                                if not self.config.keep_linebreaks:
                                      SCREAMING_SNAKE_CASE			:   Optional[Any]	  = [line.rstrip("""\n"""		) for line in batch]
                                SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = pa.Table.from_arrays([pa.array(UpperCAmelCase__		)]					,					names=UpperCAmelCase__		)
                                # Uncomment for debugging (will print the Arrow table size and elements)
                                # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                yield (file_idx, batch_idx), self._cast_table(UpperCAmelCase__		)
                                batch_idx += 1
                    elif self.config.sample_by == "paragraph":
                          SCREAMING_SNAKE_CASE			:   Any	  = 0
                          SCREAMING_SNAKE_CASE			:   Optional[int]	  = """"""
                          while True:
                                SCREAMING_SNAKE_CASE			:   Tuple	  = f.read(self.config.chunksize		)
                                if not new_batch:
                                      break
                                batch += new_batch
                                batch += f.readline()  # finish current line
                                SCREAMING_SNAKE_CASE			:   Optional[Any]	  = batch.split("""\n\n"""		)
                                SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = pa.Table.from_arrays(
                                    [pa.array([example for example in batch[:-1] if example]		)]					,					names=UpperCAmelCase__		)
                                # Uncomment for debugging (will print the Arrow table size and elements)
                                # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                yield (file_idx, batch_idx), self._cast_table(UpperCAmelCase__		)
                                batch_idx += 1
                                SCREAMING_SNAKE_CASE			:   List[Any]	  = batch[-1]
                          if batch:
                                SCREAMING_SNAKE_CASE			:   List[Any]	  = pa.Table.from_arrays([pa.array([batch]		)]					,					names=UpperCAmelCase__		)
                                yield (file_idx, batch_idx), self._cast_table(UpperCAmelCase__		)
                    elif self.config.sample_by == "document":
                          SCREAMING_SNAKE_CASE			:   List[Any]	  = f.read()
                          SCREAMING_SNAKE_CASE			:   Dict	  = pa.Table.from_arrays([pa.array([text]		)]					,					names=UpperCAmelCase__		)
                          yield file_idx, self._cast_table(UpperCAmelCase__		)







import safs







class 							a__							( safs.SaFileSystem							):







  """simple docstring"""




