






import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


A_			:Optional[int]	    =		datasets.utils.logging.get_logger(__name__)




@dataclass
class 							__A	(     datasets.BuilderConfig					):





				"""simple docstring"""







				UpperCamelCase__					: Optional[datasets.Features]     					=None
				UpperCamelCase__					: str     					="utf-8"
				UpperCamelCase__					: Optional[str]     					=None
				UpperCamelCase__					: int     					=1_0 << 2_0  # 10MB
				UpperCamelCase__					: bool     					=False
				UpperCamelCase__					: str     					="line"




class 							__A	(     datasets.ArrowBasedBuilder					):





				"""simple docstring"""







				UpperCamelCase__					: int     					=TextConfig


				def  __lowercase				(					self       ):




									"""simple docstring"""

									return datasets.DatasetInfo(features=self.config.features       )


				def  __lowercase				(					self				,    lowerCamelCase__       ):




									"""simple docstring"""

									if not self.config.data_files:
														raise ValueError(f'At least one data file must be specified, but got data_files={self.config.data_files}'       )
									__UpperCamelCase :   List[Any]									=dl_manager.download_and_extract(self.config.data_files       )
									if isinstance(lowerCamelCase__				,    (str, list, tuple)       ):
														__UpperCamelCase :   Union[str, Any]									=data_files
														if isinstance(lowerCamelCase__				,    lowerCamelCase__       ):
																			__UpperCamelCase :   Optional[Any]									=[files]
														__UpperCamelCase :   Dict									=[dl_manager.iter_files(lowerCamelCase__       ) for file in files]
														return [datasets.SplitGenerator(name=datasets.Split.TRAIN				,    gen_kwargs={'files': files}       )]
									__UpperCamelCase :   Optional[Any]									=[]
									for split_name, files in data_files.items():
														if isinstance(lowerCamelCase__				,    lowerCamelCase__       ):
																			__UpperCamelCase :   List[Any]									=[files]
														__UpperCamelCase :   Optional[Any]									=[dl_manager.iter_files(lowerCamelCase__       ) for file in files]
														splits.append(datasets.SplitGenerator(name=lowerCamelCase__				,    gen_kwargs={'files': files}       )       )
									return splits


				def  __lowercase				(					self				,    lowerCamelCase__       ):




									"""simple docstring"""

									if self.config.features is not None:
														__UpperCamelCase :   List[Any]									=self.config.features.arrow_schema
														if all(not require_storage_cast(lowerCamelCase__       ) for feature in self.config.features.values()       ):
																			# cheaper cast
																			__UpperCamelCase :   Optional[Any]									=pa_table.cast(lowerCamelCase__       )
														else:
																			# more expensive cast; allows str <-> int/float or str to Audio for example
																			__UpperCamelCase :   List[Any]									=table_cast(lowerCamelCase__				,    lowerCamelCase__       )
														return pa_table
									else:
														return pa_table.cast(pa.schema({'text': pa.string()}       )       )







				def  __lowercase				(					self				,    lowerCamelCase__       ):




									"""simple docstring"""

									__UpperCamelCase :   Optional[int]									=list(self.config.features       ) if self.config.features is not None else ['text']
									for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCamelCase__       )       ):
														# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
														with open(lowerCamelCase__				,    encoding=self.config.encoding				,    errors=self.config.errors       ) as f:
																			if self.config.sample_by == "line":
																								__UpperCamelCase :   List[str]									=0
																								while True:
																													__UpperCamelCase :   Tuple									=f.read(self.config.chunksize       )
																													if not batch:
																																		break
																													batch += f.readline()  # finish current line
																													# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																													__UpperCamelCase :   int									=StringIO(lowerCamelCase__       ).readlines()
																													if not self.config.keep_linebreaks:
																																		__UpperCamelCase :   Optional[int]									=[line.rstrip('\n'       ) for line in batch]
																													__UpperCamelCase :   List[Any]									=pa.Table.from_arrays([pa.array(lowerCamelCase__       )]				,    names=lowerCamelCase__       )
																													# Uncomment for debugging (will print the Arrow table size and elements)
																													# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																													# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																													yield (file_idx, batch_idx), self._cast_table(lowerCamelCase__       )
																													batch_idx += 1
																			elif self.config.sample_by == "paragraph":
																								__UpperCamelCase :   Any									=0
																								__UpperCamelCase :   Optional[int]									=''
																								while True:
																													__UpperCamelCase :   List[str]									=f.read(self.config.chunksize       )
																													if not new_batch:
																																		break
																													batch += new_batch
																													batch += f.readline()  # finish current line
																													__UpperCamelCase :   Any									=batch.split('\n\n'       )
																													__UpperCamelCase :   Dict									=pa.Table.from_arrays(
																													    [pa.array([example for example in batch[:-1] if example]       )]				,    names=lowerCamelCase__       )
																													# Uncomment for debugging (will print the Arrow table size and elements)
																													# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																													# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																													yield (file_idx, batch_idx), self._cast_table(lowerCamelCase__       )
																													batch_idx += 1
																													__UpperCamelCase :   Optional[Any]									=batch[-1]
																								if batch:
																													__UpperCamelCase :   Tuple									=pa.Table.from_arrays([pa.array([batch]       )]				,    names=lowerCamelCase__       )
																													yield (file_idx, batch_idx), self._cast_table(lowerCamelCase__       )
																			elif self.config.sample_by == "document":
																								__UpperCamelCase :   List[Any]									=f.read()
																								__UpperCamelCase :   Tuple									=pa.Table.from_arrays([pa.array([text]       )]				,    names=lowerCamelCase__       )
																								yield file_idx, self._cast_table(lowerCamelCase__       )









from typing import Callable






def 				A	(			a_   )	->		List[str]:

					def wrapper(a_   ):
										__UpperCamelCase :   Dict									=function_with_docstring.__doc__
										return target_function

					return wrapper

