





'''simple docstring'''


import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


_lowerCamelCase					:						int					     =							datasets.utils.logging.get_logger(__name__)



@dataclass
class 				__UpperCAmelCase	(							datasets.BuilderConfig    ):







		'''simple docstring'''







		__lowerCAmelCase								=     None
		__lowerCAmelCase								=     "utf-8"
		__lowerCAmelCase								=     None
		__lowerCAmelCase								=     10 << 20  # 10MB
		__lowerCAmelCase								=     False
		__lowerCAmelCase								=     "line"






class 				__UpperCAmelCase	(							datasets.ArrowBasedBuilder    ):







		'''simple docstring'''







		__lowerCAmelCase								=     TextConfig







		def 							A  (self    :   List[Any]   ):
			return datasets.DatasetInfo(features=self.config.features   )







		def 							A  (self    :   Dict   ,   _lowerCAmelCase    :   Union[str, Any]   ):
			if not self.config.data_files:
				raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""   )
			A					=   dl_manager.download_and_extract(self.config.data_files   )
			if isinstance(_lowerCAmelCase   ,   (str, list, tuple)   ):
				A					=   data_files
				if isinstance(_lowerCAmelCase   ,   _lowerCAmelCase   ):
					A					=   [files]
				A					=   [dl_manager.iter_files(_lowerCAmelCase   ) for file in files]
				return [datasets.SplitGenerator(name=datasets.Split.TRAIN   ,   gen_kwargs={"""files""": files}   )]
			A					=   []
			for split_name, files in data_files.items():
				if isinstance(_lowerCAmelCase   ,   _lowerCAmelCase   ):
					A					=   [files]
				A					=   [dl_manager.iter_files(_lowerCAmelCase   ) for file in files]
				splits.append(datasets.SplitGenerator(name=_lowerCAmelCase   ,   gen_kwargs={"""files""": files}   )   )
			return splits







		def 							A  (self    :   List[Any]   ,   _lowerCAmelCase    :   pa.Table   ):
			if self.config.features is not None:
				A					=   self.config.features.arrow_schema
				if all(not require_storage_cast(_lowerCAmelCase   ) for feature in self.config.features.values()   ):
					# cheaper cast
					A					=   pa_table.cast(_lowerCAmelCase   )
				else:
					# more expensive cast; allows str <-> int/float or str to Audio for example
					A					=   table_cast(_lowerCAmelCase   ,   _lowerCAmelCase   )
				return pa_table
			else:
				return pa_table.cast(pa.schema({"""text""": pa.string()}   )   )







		def 							A  (self    :   Dict   ,   _lowerCAmelCase    :   Union[str, Any]   ):
			A					=   list(self.config.features   ) if self.config.features is not None else ["""text"""]
			for file_idx, file in enumerate(itertools.chain.from_iterable(_lowerCAmelCase   )   ):
				# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
				with open(_lowerCAmelCase   ,   encoding=self.config.encoding   ,   errors=self.config.errors   ) as f:
					if self.config.sample_by == "line":
						A					=   0
						while True:
							A					=   f.read(self.config.chunksize   )
							if not batch:
								break
							batch += f.readline()  # finish current line
							# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
							A					=   StringIO(_lowerCAmelCase   ).readlines()
							if not self.config.keep_linebreaks:
								A					=   [line.rstrip("""\n"""   ) for line in batch]
							A					=   pa.Table.from_arrays([pa.array(_lowerCAmelCase   )]   ,   names=_lowerCAmelCase   )
							# Uncomment for debugging (will print the Arrow table size and elements)
							# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
							# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
							yield (file_idx, batch_idx), self._cast_table(_lowerCAmelCase   )
							batch_idx += 1
					elif self.config.sample_by == "paragraph":
						A					=   0
						A					=   """"""
						while True:
							A					=   f.read(self.config.chunksize   )
							if not new_batch:
								break
							batch += new_batch
							batch += f.readline()  # finish current line
							A					=   batch.split("""\n\n"""   )
							A					=   pa.Table.from_arrays(
							    [pa.array([example for example in batch[:-1] if example]   )]   ,   names=_lowerCAmelCase   )
							# Uncomment for debugging (will print the Arrow table size and elements)
							# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
							# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
							yield (file_idx, batch_idx), self._cast_table(_lowerCAmelCase   )
							batch_idx += 1
							A					=   batch[-1]
						if batch:
							A					=   pa.Table.from_arrays([pa.array([batch]   )]   ,   names=_lowerCAmelCase   )
							yield (file_idx, batch_idx), self._cast_table(_lowerCAmelCase   )
					elif self.config.sample_by == "document":
						A					=   f.read()
						A					=   pa.Table.from_arrays([pa.array([text]   )]   ,   names=_lowerCAmelCase   )
						yield file_idx, self._cast_table(_lowerCAmelCase   )








'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


_lowerCamelCase					:						Dict					     =							{
    'configuration_donut_swin': ['DONUT_SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP', 'DonutSwinConfig'],
    'processing_donut': ['DonutProcessor'],
}

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				_lowerCamelCase					:						List[str]					     =							[
				    'DONUT_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST',
				    'DonutSwinModel',
				    'DonutSwinPreTrainedModel',
				]

try:
				if not is_vision_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				_lowerCamelCase					:						Any					     =							['DonutFeatureExtractor']
				_lowerCamelCase					:						Union[str, Any]					     =							['DonutImageProcessor']


if TYPE_CHECKING:
				from .configuration_donut_swin import DONUT_SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP, DonutSwinConfig
				from .processing_donut import DonutProcessor

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_donut_swin import (
								    DONUT_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
								    DonutSwinModel,
								    DonutSwinPreTrainedModel,
								)

				try:
								if not is_vision_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .feature_extraction_donut import DonutFeatureExtractor
								from .image_processing_donut import DonutImageProcessor

else:
				import sys

				_lowerCamelCase					:						Optional[int]					     =							_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)

