
'''simple docstring'''







import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


__SCREAMING_SNAKE_CASE		:  str        =      datasets.utils.logging.get_logger(__name__)
@dataclass
class  SCREAMING_SNAKE_CASE__       (  datasets.BuilderConfig   ):
	lowercase__	    =							None
	lowercase__	    =							"utf-8"
	lowercase__	    =							None
	lowercase__	    =							10 << 20  # 10MB
	lowercase__	    =							False
	lowercase__	    =							"line"







class  SCREAMING_SNAKE_CASE__       (  datasets.ArrowBasedBuilder   ):
	lowercase__	    =							TextConfig
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					return datasets.DatasetInfo(features=self.config.features			)
	def __lowerCamelCase    (				self  ,    __UpperCamelCase			):



					'''simple docstring'''






					if not self.config.data_files:
									raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""			)
					__a       :     Any								=	dl_manager.download_and_extract(self.config.data_files			)
					if isinstance(_A  ,    (str, list, tuple)			):
									__a       :     Optional[Any]								=	data_files
									if isinstance(_A  ,    _A			):
													__a       :     List[str]								=	[files]
									__a       :     Any								=	[dl_manager.iter_files(_A			) for file in files]
									return [datasets.SplitGenerator(name=datasets.Split.TRAIN  ,    gen_kwargs={"""files""": files}			)]
					__a       :     int								=	[]
					for split_name, files in data_files.items():
									if isinstance(_A  ,    _A			):
													__a       :     Dict								=	[files]
									__a       :     int								=	[dl_manager.iter_files(_A			) for file in files]
									splits.append(datasets.SplitGenerator(name=_A  ,    gen_kwargs={"""files""": files}			)			)
					return splits
	def __lowerCamelCase    (				self  ,    __UpperCamelCase			):



					'''simple docstring'''






					if self.config.features is not None:
									__a       :     Tuple								=	self.config.features.arrow_schema
									if all(not require_storage_cast(_A			) for feature in self.config.features.values()			):
													# cheaper cast
													__a       :     List[Any]								=	pa_table.cast(_A			)
									else:
													# more expensive cast; allows str <-> int/float or str to Audio for example
													__a       :     Tuple								=	table_cast(_A  ,    _A			)
									return pa_table
					else:
									return pa_table.cast(pa.schema({"""text""": pa.string()}			)			)

	def __lowerCamelCase    (				self  ,    __UpperCamelCase			):



					'''simple docstring'''






					__a       :     Tuple								=	list(self.config.features			) if self.config.features is not None else ['text']
					for file_idx, file in enumerate(itertools.chain.from_iterable(_A			)			):
									# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
									with open(_A  ,    encoding=self.config.encoding  ,    errors=self.config.errors			) as f:
													if self.config.sample_by == "line":
																	__a       :     Tuple								=	0
																	while True:
																					__a       :     int								=	f.read(self.config.chunksize			)
																					if not batch:
																									break
																					batch += f.readline()  # finish current line
																					# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																					__a       :     Union[str, Any]								=	StringIO(_A			).readlines()
																					if not self.config.keep_linebreaks:
																									__a       :     Tuple								=	[line.rstrip("""\n"""			) for line in batch]
																					__a       :     str								=	pa.Table.from_arrays([pa.array(_A			)]  ,    names=_A			)
																					# Uncomment for debugging (will print the Arrow table size and elements)
																					# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																					# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																					yield (file_idx, batch_idx), self._cast_table(_A			)
																					batch_idx += 1
													elif self.config.sample_by == "paragraph":
																	__a       :     List[str]								=	0
																	__a       :     Union[str, Any]								=	''
																	while True:
																					__a       :     Dict								=	f.read(self.config.chunksize			)
																					if not new_batch:
																									break
																					batch += new_batch
																					batch += f.readline()  # finish current line
																					__a       :     Union[str, Any]								=	batch.split("""\n\n"""			)
																					__a       :     Dict								=	pa.Table.from_arrays(
																					    [pa.array([example for example in batch[:-1] if example]			)]  ,    names=_A			)
																					# Uncomment for debugging (will print the Arrow table size and elements)
																					# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																					# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																					yield (file_idx, batch_idx), self._cast_table(_A			)
																					batch_idx += 1
																					__a       :     List[str]								=	batch[-1]
																	if batch:
																					__a       :     int								=	pa.Table.from_arrays([pa.array([batch]			)]  ,    names=_A			)
																					yield (file_idx, batch_idx), self._cast_table(_A			)
													elif self.config.sample_by == "document":
																	__a       :     Tuple								=	f.read()
																	__a       :     Union[str, Any]								=	pa.Table.from_arrays([pa.array([text]			)]  ,    names=_A			)
																	yield file_idx, self._cast_table(_A			)
"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


_a	:	Optional[Any]=					{"configuration_gptj": ["GPTJ_PRETRAINED_CONFIG_ARCHIVE_MAP", "GPTJConfig", "GPTJOnnxConfig"]}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_a	:	Dict=					[
	    "GPTJ_PRETRAINED_MODEL_ARCHIVE_LIST",
	    "GPTJForCausalLM",
	    "GPTJForQuestionAnswering",
	    "GPTJForSequenceClassification",
	    "GPTJModel",
	    "GPTJPreTrainedModel",
	]

try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_a	:	Optional[Any]=					[
	    "TFGPTJForCausalLM",
	    "TFGPTJForQuestionAnswering",
	    "TFGPTJForSequenceClassification",
	    "TFGPTJModel",
	    "TFGPTJPreTrainedModel",
	]

try:
	if not is_flax_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_a	:	List[str]=					[
	    "FlaxGPTJForCausalLM",
	    "FlaxGPTJModel",
	    "FlaxGPTJPreTrainedModel",
	]


if TYPE_CHECKING:
	from .configuration_gptj import GPTJ_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTJConfig, GPTJOnnxConfig

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_gptj import (
		    GPTJ_PRETRAINED_MODEL_ARCHIVE_LIST,
		    GPTJForCausalLM,
		    GPTJForQuestionAnswering,
		    GPTJForSequenceClassification,
		    GPTJModel,
		    GPTJPreTrainedModel,
		)

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_tf_gptj import (
		    TFGPTJForCausalLM,
		    TFGPTJForQuestionAnswering,
		    TFGPTJForSequenceClassification,
		    TFGPTJModel,
		    TFGPTJPreTrainedModel,
		)

	try:
		if not is_flax_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_flax_gptj import FlaxGPTJForCausalLM, FlaxGPTJModel, FlaxGPTJPreTrainedModel

else:
	import sys

	_a	:	Any=					_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





