


"""simple docstring"""






import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


SCREAMING_SNAKE_CASE__				:  List[str]						= datasets.utils.logging.get_logger(__name__)


@dataclass
class 	A_       (     datasets.BuilderConfig   ):





  """simple docstring"""

  lowercase		:	Union[str, Any]   =						None
  lowercase		:	Dict   =						"utf-8"
  lowercase		:	Optional[int]   =						None
  lowercase		:	List[Any]   =						10 << 20  # 10MB
  lowercase		:	List[str]   =						False
  lowercase		:	Dict   =						"line"



class 	A_       (     datasets.ArrowBasedBuilder   ):





  """simple docstring"""

  lowercase		:	List[str]   =						TextConfig
  def 							lowercase_     (						self						)				->					List[Any]:
     return datasets.DatasetInfo(features=self.config.features						)
  def 							lowercase_     (						self  ,					__UpperCAmelCase						)				->					Tuple:






     if not self.config.data_files:
        raise ValueError(f'At least one data file must be specified, but got data_files={self.config.data_files}'						)
     a   :	List[str]						=    dl_manager.download_and_extract(self.config.data_files						)
     if isinstance(__UpperCAmelCase  ,					(str, list, tuple)						):
        a   :	Union[str, Any]						=    data_files
        if isinstance(__UpperCAmelCase  ,					__UpperCAmelCase						):
           a   :	Optional[int]						=    [files]
        a   :	List[Any]						=    [dl_manager.iter_files(__UpperCAmelCase						) for file in files]
        return [datasets.SplitGenerator(name=datasets.Split.TRAIN  ,					gen_kwargs={'files': files}						)]
     a   :	Union[str, Any]						=    []
     for split_name, files in data_files.items():
        if isinstance(__UpperCAmelCase  ,					__UpperCAmelCase						):
           a   :	str						=    [files]
        a   :	str						=    [dl_manager.iter_files(__UpperCAmelCase						) for file in files]
        splits.append(datasets.SplitGenerator(name=__UpperCAmelCase  ,					gen_kwargs={'files': files}						)						)
     return splits
  def 							lowercase_     (						self  ,					__UpperCAmelCase						)				->					pa.Table:






     if self.config.features is not None:
        a   :	List[str]						=    self.config.features.arrow_schema
        if all(not require_storage_cast(__UpperCAmelCase						) for feature in self.config.features.values()						):
           # cheaper cast
           a   :	Optional[int]						=    pa_table.cast(__UpperCAmelCase						)
        else:
           # more expensive cast; allows str <-> int/float or str to Audio for example
           a   :	List[Any]						=    table_cast(__UpperCAmelCase  ,					__UpperCAmelCase						)
        return pa_table
     else:
        return pa_table.cast(pa.schema({'text': pa.string()}						)						)






  def 							lowercase_     (						self  ,					__UpperCAmelCase						)				->					Tuple:
     a   :	Dict						=    list(self.config.features						) if self.config.features is not None else ['text']
     for file_idx, file in enumerate(itertools.chain.from_iterable(__UpperCAmelCase						)						):
        # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
        with open(__UpperCAmelCase  ,					encoding=self.config.encoding  ,					errors=self.config.errors						) as f:
           if self.config.sample_by == "line":
              a   :	Optional[Any]						=    0
              while True:
                 a   :	str						=    f.read(self.config.chunksize						)
                 if not batch:
                    break
                 batch += f.readline()  # finish current line
                 # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
                 a   :	int						=    StringIO(__UpperCAmelCase						).readlines()
                 if not self.config.keep_linebreaks:
                    a   :	Union[str, Any]						=    [line.rstrip('\n'						) for line in batch]
                 a   :	List[Any]						=    pa.Table.from_arrays([pa.array(__UpperCAmelCase						)]  ,					names=__UpperCAmelCase						)
                 # Uncomment for debugging (will print the Arrow table size and elements)
                 # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                 # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                 yield (file_idx, batch_idx), self._cast_table(__UpperCAmelCase						)
                 batch_idx += 1
           elif self.config.sample_by == "paragraph":
              a   :	Any						=    0
              a   :	str						=    ''
              while True:
                 a   :	Dict						=    f.read(self.config.chunksize						)
                 if not new_batch:
                    break
                 batch += new_batch
                 batch += f.readline()  # finish current line
                 a   :	Optional[int]						=    batch.split('\n\n'						)
                 a   :	int						=    pa.Table.from_arrays(
                     [pa.array([example for example in batch[:-1] if example]						)]  ,					names=__UpperCAmelCase						)
                 # Uncomment for debugging (will print the Arrow table size and elements)
                 # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                 # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                 yield (file_idx, batch_idx), self._cast_table(__UpperCAmelCase						)
                 batch_idx += 1
                 a   :	Tuple						=    batch[-1]
              if batch:
                 a   :	List[str]						=    pa.Table.from_arrays([pa.array([batch]						)]  ,					names=__UpperCAmelCase						)
                 yield (file_idx, batch_idx), self._cast_table(__UpperCAmelCase						)
           elif self.config.sample_by == "document":
              a   :	Dict						=    f.read()
              a   :	Dict						=    pa.Table.from_arrays([pa.array([text]						)]  ,					names=__UpperCAmelCase						)
              yield file_idx, self._cast_table(__UpperCAmelCase						)










from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


UpperCAmelCase__  =			{
    "configuration_align": [
        "ALIGN_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "AlignConfig",
        "AlignTextConfig",
        "AlignVisionConfig",
    ],
    "processing_align": ["AlignProcessor"],
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							UpperCAmelCase__  =			[
							    "ALIGN_PRETRAINED_MODEL_ARCHIVE_LIST",
							    "AlignModel",
							    "AlignPreTrainedModel",
							    "AlignTextModel",
							    "AlignVisionModel",
							]

if TYPE_CHECKING:
							from .configuration_align import (
							    ALIGN_PRETRAINED_CONFIG_ARCHIVE_MAP,
							    AlignConfig,
							    AlignTextConfig,
							    AlignVisionConfig,
							)
							from .processing_align import AlignProcessor

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_align import (
														    ALIGN_PRETRAINED_MODEL_ARCHIVE_LIST,
														    AlignModel,
														    AlignPreTrainedModel,
														    AlignTextModel,
														    AlignVisionModel,
														)

else:
							import sys

							UpperCAmelCase__  =			_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
