import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


__snake_case         =     datasets.utils.logging.get_logger(__name__)







@dataclass
class       __snake_case (  datasets.BuilderConfig ):
 __lowerCamelCase     : Optional[datasets.Features] 	=   None
 __lowerCamelCase     : str 	=   "utf-8"
 __lowerCamelCase     : Optional[str] 	=   None
 __lowerCamelCase     : int 	=   10 << 20  # 10MB
 __lowerCamelCase     : bool 	=   False
 __lowerCamelCase     : str 	=   "line"







class       __snake_case (  datasets.ArrowBasedBuilder ):
 __lowerCamelCase     : int 	=   TextConfig
 def 						UpperCAmelCase__							(	self    )						->				Dict:
       '''simple docstring'''





       return datasets.DatasetInfo(features=self.config.features    )
 def 						UpperCAmelCase__							(	self		, snake_case__    )						->				str:
       '''simple docstring'''





       if not self.config.data_files:
             raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}'''    )
       UpperCAmelCase     :			List[Any]     	=dl_manager.download_and_extract(self.config.data_files    )
       if isinstance(lowerCamelCase__		, (str, list, tuple)    ):
             UpperCAmelCase     :			Union[str, Any]     	=data_files
             if isinstance(lowerCamelCase__		, lowerCamelCase__    ):
                   UpperCAmelCase     :			Optional[Any]     	=[files]
             UpperCAmelCase     :			Dict     	=[dl_manager.iter_files(lowerCamelCase__    ) for file in files]
             return [datasets.SplitGenerator(name=datasets.Split.TRAIN		, gen_kwargs={'''files''': files}    )]
       UpperCAmelCase     :			Optional[Any]     	=[]
       for split_name, files in data_files.items():
             if isinstance(lowerCamelCase__		, lowerCamelCase__    ):
                   UpperCAmelCase     :			List[Any]     	=[files]
             UpperCAmelCase     :			Optional[Any]     	=[dl_manager.iter_files(lowerCamelCase__    ) for file in files]
             splits.append(datasets.SplitGenerator(name=lowerCamelCase__		, gen_kwargs={'''files''': files}    )    )
       return splits
 def 						UpperCAmelCase__							(	self		, snake_case__    )						->				Any:
       '''simple docstring'''





       if self.config.features is not None:
             UpperCAmelCase     :			List[Any]     	=self.config.features.arrow_schema
             if all(not require_storage_cast(lowerCamelCase__    ) for feature in self.config.features.values()    ):
                   # cheaper cast
                   UpperCAmelCase     :			Optional[Any]     	=pa_table.cast(lowerCamelCase__    )
             else:
                   # more expensive cast; allows str <-> int/float or str to Audio for example
                   UpperCAmelCase     :			List[Any]     	=table_cast(lowerCamelCase__		, lowerCamelCase__    )
             return pa_table
       else:
             return pa_table.cast(pa.schema({'''text''': pa.string()}    )    )






 def 						UpperCAmelCase__							(	self		, snake_case__    )						->				Union[str, Any]:
       '''simple docstring'''





       UpperCAmelCase     :			Optional[int]     	=list(self.config.features    ) if self.config.features is not None else ['text']
       for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCamelCase__    )    ):
             # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
             with open(lowerCamelCase__		, encoding=self.config.encoding		, errors=self.config.errors    ) as f:
                   if self.config.sample_by == "line":
                         UpperCAmelCase     :			List[str]     	=0
                         while True:
                               UpperCAmelCase     :			Tuple     	=f.read(self.config.chunksize    )
                               if not batch:
                                     break
                               batch += f.readline()  # finish current line
                               # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
                               UpperCAmelCase     :			int     	=StringIO(lowerCamelCase__    ).readlines()
                               if not self.config.keep_linebreaks:
                                     UpperCAmelCase     :			Optional[int]     	=[line.rstrip('''\n'''    ) for line in batch]
                               UpperCAmelCase     :			List[Any]     	=pa.Table.from_arrays([pa.array(lowerCamelCase__    )]		, names=lowerCamelCase__    )
                               # Uncomment for debugging (will print the Arrow table size and elements)
                               # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                               # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                               yield (file_idx, batch_idx), self._cast_table(lowerCamelCase__    )
                               batch_idx += 1
                   elif self.config.sample_by == "paragraph":
                         UpperCAmelCase     :			Any     	=0
                         UpperCAmelCase     :			Optional[int]     	=''
                         while True:
                               UpperCAmelCase     :			List[str]     	=f.read(self.config.chunksize    )
                               if not new_batch:
                                     break
                               batch += new_batch
                               batch += f.readline()  # finish current line
                               UpperCAmelCase     :			Any     	=batch.split('''\n\n'''    )
                               UpperCAmelCase     :			Dict     	=pa.Table.from_arrays(
                                   [pa.array([example for example in batch[:-1] if example]    )]		, names=lowerCamelCase__    )
                               # Uncomment for debugging (will print the Arrow table size and elements)
                               # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                               # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                               yield (file_idx, batch_idx), self._cast_table(lowerCamelCase__    )
                               batch_idx += 1
                               UpperCAmelCase     :			Optional[Any]     	=batch[-1]
                         if batch:
                               UpperCAmelCase     :			Tuple     	=pa.Table.from_arrays([pa.array([batch]    )]		, names=lowerCamelCase__    )
                               yield (file_idx, batch_idx), self._cast_table(lowerCamelCase__    )
                   elif self.config.sample_by == "document":
                         UpperCAmelCase     :			List[Any]     	=f.read()
                         UpperCAmelCase     :			Tuple     	=pa.Table.from_arrays([pa.array([text]    )]		, names=lowerCamelCase__    )
                         yield file_idx, self._cast_table(lowerCamelCase__    )








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


A_			:Union[str, Any]	    =		{'''configuration_nat''': ['''NAT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''NatConfig''']}


try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							A_			:Optional[Any]	    =		[
							    '''NAT_PRETRAINED_MODEL_ARCHIVE_LIST''',
							    '''NatForImageClassification''',
							    '''NatModel''',
							    '''NatPreTrainedModel''',
							    '''NatBackbone''',
							]

if TYPE_CHECKING:
							from .configuration_nat import NAT_PRETRAINED_CONFIG_ARCHIVE_MAP, NatConfig

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_nat import (
														    NAT_PRETRAINED_MODEL_ARCHIVE_LIST,
														    NatBackbone,
														    NatForImageClassification,
														    NatModel,
														    NatPreTrainedModel,
														)

else:
							import sys

							A_			:Tuple	    =		_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)

