





'''simple docstring'''






import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


__UpperCamelCase      :   Union[str, Any]								=     datasets.utils.logging.get_logger(__name__)



@dataclass
class  lowerCamelCase__				(						datasets.BuilderConfig						):


	"""simple docstring"""





	__magic_name__							   =		None
	__magic_name__							   =		"utf-8"
	__magic_name__							   =		None
	__magic_name__							   =		1_0 << 2_0  # 10MB
	__magic_name__							   =		False
	__magic_name__							   =		"line"



class  lowerCamelCase__				(						datasets.ArrowBasedBuilder						):


	"""simple docstring"""





	__magic_name__							   =		TextConfig




	def 				_lowerCamelCase				(					self			)       ->				List[str]:
					return datasets.DatasetInfo(features=self.config.features			)




	def 				_lowerCamelCase				(					self   ,     UpperCAmelCase__			)       ->				Tuple:





					if not self.config.data_files:
									raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""			)
					_A :				Dict				=  dl_manager.download_and_extract(self.config.data_files			)
					if isinstance(__UpperCamelCase   ,     (str, list, tuple)			):
									_A :				Optional[Any]				=  data_files
									if isinstance(__UpperCamelCase   ,     __UpperCamelCase			):
													_A :				List[str]				=  [files]
									_A :				List[Any]				=  [dl_manager.iter_files(__UpperCamelCase			) for file in files]
									return [datasets.SplitGenerator(name=datasets.Split.TRAIN   ,     gen_kwargs={'''files''': files}			)]
					_A :				Optional[Any]				=  []
					for split_name, files in data_files.items():
									if isinstance(__UpperCamelCase   ,     __UpperCamelCase			):
													_A :				List[Any]				=  [files]
									_A :				Optional[int]				=  [dl_manager.iter_files(__UpperCamelCase			) for file in files]
									splits.append(datasets.SplitGenerator(name=__UpperCamelCase   ,     gen_kwargs={'''files''': files}			)			)
					return splits




	def 				_lowerCamelCase				(					self   ,     UpperCAmelCase__			)       ->				Optional[Any]:





					if self.config.features is not None:
									_A :				Dict				=  self.config.features.arrow_schema
									if all(not require_storage_cast(__UpperCamelCase			) for feature in self.config.features.values()			):
													# cheaper cast
													_A :				Optional[int]				=  pa_table.cast(__UpperCamelCase			)
									else:
													# more expensive cast; allows str <-> int/float or str to Audio for example
													_A :				List[Any]				=  table_cast(__UpperCamelCase   ,     __UpperCamelCase			)
									return pa_table
					else:
									return pa_table.cast(pa.schema({'''text''': pa.string()}			)			)




	def 				_lowerCamelCase				(					self   ,     UpperCAmelCase__			)       ->				Optional[Any]:
					_A :				Dict				=  list(self.config.features			) if self.config.features is not None else ['''text''']
					for file_idx, file in enumerate(itertools.chain.from_iterable(__UpperCamelCase			)			):
									# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
									with open(__UpperCamelCase   ,     encoding=self.config.encoding   ,     errors=self.config.errors			) as f:
													if self.config.sample_by == "line":
																	_A :				Dict				=  0
																	while True:
																					_A :				Union[str, Any]				=  f.read(self.config.chunksize			)
																					if not batch:
																									break
																					batch += f.readline()  # finish current line
																					# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																					_A :				Any				=  StringIO(__UpperCamelCase			).readlines()
																					if not self.config.keep_linebreaks:
																									_A :				Optional[Any]				=  [line.rstrip('''\n'''			) for line in batch]
																					_A :				str				=  pa.Table.from_arrays([pa.array(__UpperCamelCase			)]   ,     names=__UpperCamelCase			)
																					# Uncomment for debugging (will print the Arrow table size and elements)
																					# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																					# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																					yield (file_idx, batch_idx), self._cast_table(__UpperCamelCase			)
																					batch_idx += 1
													elif self.config.sample_by == "paragraph":
																	_A :				Dict				=  0
																	_A :				List[str]				=  ''''''
																	while True:
																					_A :				List[Any]				=  f.read(self.config.chunksize			)
																					if not new_batch:
																									break
																					batch += new_batch
																					batch += f.readline()  # finish current line
																					_A :				int				=  batch.split('''\n\n'''			)
																					_A :				Union[str, Any]				=  pa.Table.from_arrays(
																					    [pa.array([example for example in batch[:-1] if example]			)]   ,     names=__UpperCamelCase			)
																					# Uncomment for debugging (will print the Arrow table size and elements)
																					# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																					# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																					yield (file_idx, batch_idx), self._cast_table(__UpperCamelCase			)
																					batch_idx += 1
																					_A :				Any				=  batch[-1]
																	if batch:
																					_A :				Tuple				=  pa.Table.from_arrays([pa.array([batch]			)]   ,     names=__UpperCamelCase			)
																					yield (file_idx, batch_idx), self._cast_table(__UpperCamelCase			)
													elif self.config.sample_by == "document":
																	_A :				Dict				=  f.read()
																	_A :				List[Any]				=  pa.Table.from_arrays([pa.array([text]			)]   ,     names=__UpperCamelCase			)
																	yield file_idx, self._cast_table(__UpperCamelCase			)



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


A__							=				{"configuration_pegasus": ["PEGASUS_PRETRAINED_CONFIG_ARCHIVE_MAP", "PegasusConfig"]}

try:
						if not is_sentencepiece_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A__							=				["PegasusTokenizer"]

try:
						if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A__							=				["PegasusTokenizerFast"]

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A__							=				[
						    "PEGASUS_PRETRAINED_MODEL_ARCHIVE_LIST",
						    "PegasusForCausalLM",
						    "PegasusForConditionalGeneration",
						    "PegasusModel",
						    "PegasusPreTrainedModel",
						]

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A__							=				[
						    "TFPegasusForConditionalGeneration",
						    "TFPegasusModel",
						    "TFPegasusPreTrainedModel",
						]

try:
						if not is_flax_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A__							=				[
						    "FlaxPegasusForConditionalGeneration",
						    "FlaxPegasusModel",
						    "FlaxPegasusPreTrainedModel",
						]


if TYPE_CHECKING:
						from .configuration_pegasus import PEGASUS_PRETRAINED_CONFIG_ARCHIVE_MAP, PegasusConfig

						try:
												if not is_sentencepiece_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_pegasus import PegasusTokenizer

						try:
												if not is_tokenizers_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_pegasus_fast import PegasusTokenizerFast

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_pegasus import (
												    PEGASUS_PRETRAINED_MODEL_ARCHIVE_LIST,
												    PegasusForCausalLM,
												    PegasusForConditionalGeneration,
												    PegasusModel,
												    PegasusPreTrainedModel,
												)

						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_tf_pegasus import TFPegasusForConditionalGeneration, TFPegasusModel, TFPegasusPreTrainedModel

						try:
												if not is_flax_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_flax_pegasus import (
												    FlaxPegasusForConditionalGeneration,
												    FlaxPegasusModel,
												    FlaxPegasusPreTrainedModel,
												)

else:
						import sys

						A__							=				_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)




