



import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


A_		:  Tuple										=  datasets.utils.logging.get_logger(__name__)



@dataclass
class 		_lowercase		(							datasets.BuilderConfig						):
   _UpperCAmelCase  						=      None
   _UpperCAmelCase  						=      "utf-8"
   _UpperCAmelCase  						=      None
   _UpperCAmelCase  						=      10 << 20  # 10MB
   _UpperCAmelCase  						=      False
   _UpperCAmelCase  						=      "line"



class 		_lowercase		(							datasets.ArrowBasedBuilder						):
   _UpperCAmelCase  						=      TextConfig

   def 		A				(   self      :							List[Any]				)							->					Union[str, Any]:


    """simple docstring"""






    return datasets.DatasetInfo(features=self.config.features				)

   def 		A				(   self      :							str				, __lowerCAmelCase      :							int				)							->					Union[str, Any]:


    """simple docstring"""






    if not self.config.data_files:
     raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""				)
    a							=		dl_manager.download_and_extract(self.config.data_files				)
    if isinstance(__lowerCAmelCase				, (str, list, tuple)				):
     a							=		data_files
     if isinstance(__lowerCAmelCase				, __lowerCAmelCase				):
      a							=		[files]
     a							=		[dl_manager.iter_files(__lowerCAmelCase				) for file in files]
     return [datasets.SplitGenerator(name=datasets.Split.TRAIN				, gen_kwargs={"files": files}				)]
    a							=		[]
    for split_name, files in data_files.items():
     if isinstance(__lowerCAmelCase				, __lowerCAmelCase				):
      a							=		[files]
     a							=		[dl_manager.iter_files(__lowerCAmelCase				) for file in files]
     splits.append(datasets.SplitGenerator(name=__lowerCAmelCase				, gen_kwargs={"files": files}				)				)
    return splits

   def 		A				(   self      :							Optional[int]				, __lowerCAmelCase      :							pa.Table				)							->					pa.Table:


    """simple docstring"""






    if self.config.features is not None:
     a							=		self.config.features.arrow_schema
     if all(not require_storage_cast(__lowerCAmelCase				) for feature in self.config.features.values()				):
      # cheaper cast
      a							=		pa_table.cast(__lowerCAmelCase				)
     else:
      # more expensive cast; allows str <-> int/float or str to Audio for example
      a							=		table_cast(__lowerCAmelCase				, __lowerCAmelCase				)
     return pa_table
    else:
     return pa_table.cast(pa.schema({"text": pa.string()}				)				)




   def 		A				(   self      :							int				, __lowerCAmelCase      :							str				)							->					List[Any]:


    """simple docstring"""






    a							=		list(self.config.features				) if self.config.features is not None else ["text"]
    for file_idx, file in enumerate(itertools.chain.from_iterable(__lowerCAmelCase				)				):
     # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
     with open(__lowerCAmelCase				, encoding=self.config.encoding				, errors=self.config.errors				) as f:
      if self.config.sample_by == "line":
       a							=		0
       while True:
        a							=		f.read(self.config.chunksize				)
        if not batch:
         break
        batch += f.readline()  # finish current line
        # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
        a							=		StringIO(__lowerCAmelCase				).readlines()
        if not self.config.keep_linebreaks:
         a							=		[line.rstrip("\n"				) for line in batch]
        a							=		pa.Table.from_arrays([pa.array(__lowerCAmelCase				)]				, names=__lowerCAmelCase				)
        # Uncomment for debugging (will print the Arrow table size and elements)
        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
        yield (file_idx, batch_idx), self._cast_table(__lowerCAmelCase				)
        batch_idx += 1
      elif self.config.sample_by == "paragraph":
       a							=		0
       a							=		""
       while True:
        a							=		f.read(self.config.chunksize				)
        if not new_batch:
         break
        batch += new_batch
        batch += f.readline()  # finish current line
        a							=		batch.split("\n\n"				)
        a							=		pa.Table.from_arrays(
            [pa.array([example for example in batch[:-1] if example]				)]				, names=__lowerCAmelCase				)
        # Uncomment for debugging (will print the Arrow table size and elements)
        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
        yield (file_idx, batch_idx), self._cast_table(__lowerCAmelCase				)
        batch_idx += 1
        a							=		batch[-1]
       if batch:
        a							=		pa.Table.from_arrays([pa.array([batch]				)]				, names=__lowerCAmelCase				)
        yield (file_idx, batch_idx), self._cast_table(__lowerCAmelCase				)
      elif self.config.sample_by == "document":
       a							=		f.read()
       a							=		pa.Table.from_arrays([pa.array([text]				)]				, names=__lowerCAmelCase				)
       yield file_idx, self._cast_table(__lowerCAmelCase				)







'''simple docstring'''




import abc
import copy
import dataclasses
from dataclasses import dataclass
from typing import ClassVar, Dict, Type, TypeVar

from ..features import Features


snake_case__ 							=    TypeVar("""T""", bound="""TaskTemplate""")






@dataclass(frozen=a__  )
class      UpperCamelCase_  (abc.ABC  ):







						"""simple docstring"""


						_lowerCAmelCase         =       42
						_lowerCAmelCase         =       42
						_lowerCAmelCase         =       42







						def    _a     (  self   :							T				,			_lowerCamelCase   :							Features     ):





											"""simple docstring"""







											return copy.deepcopy(self     )







						@property
						def    _a     (  self   :							List[str]     ):





											"""simple docstring"""







											return Features(**self.input_schema				,			**self.label_schema     )







						@property
						@abc.abstractmethod
						def    _a     (  self   :							int     ):





											"""simple docstring"""







											raise NotImplementedError







						@classmethod
						def    _a     (  cls   :							Type[T]				,			_lowerCamelCase   :							dict     ):





											"""simple docstring"""







											A_			:  Dict			=				{f.name for f in dataclasses.fields(cls     )}
											return cls(**{k: v for k, v in template_dict.items() if k in field_names}     )



