
"""simple docstring"""






import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader




class      _SCREAMING_SNAKE_CASE   (     A__				):






			def __init__(   self			,  __A			,  __A = None			,  __A = None			,  __A = None			,  __A = False			,  __A = False			,  __A = None			,  **__A			,  )     ->	str:
						super().__init__(
						    __A			,  split=__A			,  features=__A			,  cache_dir=__A			,  keep_in_memory=__A			,  streaming=__A			,  num_proc=__A			,  **__A			,  )
						lowerCAmelCase_       :Dict     	=    path_or_paths if isinstance(__A			,  __A			) else {self.split: path_or_paths}
						lowerCAmelCase_       :Dict     	=    Csv(
						    cache_dir=__A			,  data_files=__A			,  features=__A			,  **__A			,  )






			def 					__lowerCAmelCase					(   self			)     ->	Union[str, Any]:
						# Build iterable dataset
						if self.streaming:
									lowerCAmelCase_       :Optional[Any]     	=    self.builder.as_streaming_dataset(split=self.split			)
						# Build regular (map-style) dataset
						else:
									lowerCAmelCase_       :Any     	=    None
									lowerCAmelCase_       :Optional[int]     	=    None
									lowerCAmelCase_       :List[Any]     	=    None
									lowerCAmelCase_       :Any     	=    None

									self.builder.download_and_prepare(
									    download_config=__A			,  download_mode=__A			,  verification_mode=__A			,  base_path=__A			,  num_proc=self.num_proc			,  )
									lowerCAmelCase_       :str     	=    self.builder.as_dataset(
									    split=self.split			,  verification_mode=__A			,  in_memory=self.keep_in_memory			)
						return dataset




class      _SCREAMING_SNAKE_CASE   :






			def __init__(   self			,  __A			,  __A			,  __A = None			,  __A = None			,  **__A			,  )     ->	Optional[int]:
						if num_proc is not None and num_proc <= 0:
									raise ValueError(f"""num_proc {num_proc} must be an integer > 0."""			)

						lowerCAmelCase_       :Optional[int]     	=    dataset
						lowerCAmelCase_       :Optional[Any]     	=    path_or_buf
						lowerCAmelCase_       :Dict     	=    batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
						lowerCAmelCase_       :Dict     	=    num_proc
						lowerCAmelCase_       :Any     	=    """utf-8"""
						lowerCAmelCase_       :Tuple     	=    to_csv_kwargs






			def 					__lowerCAmelCase					(   self			)     ->	int:
						lowerCAmelCase_       :Any     	=    self.to_csv_kwargs.pop("""path_or_buf"""			,  __A			)
						lowerCAmelCase_       :int     	=    self.to_csv_kwargs.pop("""header"""			,  __A			)
						lowerCAmelCase_       :int     	=    self.to_csv_kwargs.pop("""index"""			,  __A			)

						if isinstance(self.path_or_buf			,  (str, bytes, os.PathLike)			):
									with open(self.path_or_buf			,  """wb+"""			) as buffer:
												lowerCAmelCase_       :Dict     	=    self._write(file_obj=__A			,  header=__A			,  index=__A			,  **self.to_csv_kwargs			)
						else:
									lowerCAmelCase_       :Dict     	=    self._write(file_obj=self.path_or_buf			,  header=__A			,  index=__A			,  **self.to_csv_kwargs			)
						return written






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	Tuple:
						lowerCAmelCase_	,			lowerCAmelCase_	,			lowerCAmelCase_	,			lowerCAmelCase_       :Dict     	=    args

						lowerCAmelCase_       :List[Any]     	=    query_table(
						    table=self.dataset.data			,  key=slice(__A			,  offset + self.batch_size			)			,  indices=self.dataset._indices			,  )
						lowerCAmelCase_       :Tuple     	=    batch.to_pandas().to_csv(
						    path_or_buf=__A			,  header=header if (offset == 0) else False			,  index=__A			,  **__A			)
						return csv_str.encode(self.encoding			)






			def 					__lowerCAmelCase					(   self			,  __A			,  __A			,  __A			,  **__A			)     ->	int:
						lowerCAmelCase_       :Union[str, Any]     	=    0

						if self.num_proc is None or self.num_proc == 1:
									for offset in logging.tqdm(
									    range(0			,  len(self.dataset			)			,  self.batch_size			)			,  unit="""ba"""			,  disable=not logging.is_progress_bar_enabled()			,  desc="""Creating CSV from Arrow format"""			,  ):
												lowerCAmelCase_       :Any     	=    self._batch_csv((offset, header, index, to_csv_kwargs)			)
												written += file_obj.write(__A			)

						else:
									lowerCAmelCase_	,			lowerCAmelCase_       :Optional[int]     	=    len(self.dataset			), self.batch_size
									with multiprocessing.Pool(self.num_proc			) as pool:
												for csv_str in logging.tqdm(
												    pool.imap(
												        self._batch_csv			,  [(offset, header, index, to_csv_kwargs) for offset in range(0			,  __A			,  __A			)]			,  )			,  total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size			,  unit="""ba"""			,  disable=not logging.is_progress_bar_enabled()			,  desc="""Creating CSV from Arrow format"""			,  ):
															written += file_obj.write(__A			)

						return written








"""simple docstring"""






from typing import Optional, Union

from ...file_utils import TensorType
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, TruncationStrategy




class      _SCREAMING_SNAKE_CASE   (     A__				):
			UpperCAmelCase_    :Union[str, Any]												=		"MarkupLMFeatureExtractor"
			UpperCAmelCase_    :Optional[Any]												=		("MarkupLMTokenizer", "MarkupLMTokenizerFast")
			UpperCAmelCase_    :List[Any]												=		True






			def __call__(   self			,  __A=None			,  __A=None			,  __A=None			,  __A=None			,  __A=None			,  __A = True			,  __A = False			,  __A = None			,  __A = None			,  __A = 0			,  __A = None			,  __A = None			,  __A = None			,  __A = False			,  __A = False			,  __A = False			,  __A = False			,  __A = True			,  __A = None			,  **__A			,  )     ->	BatchEncoding:
						# first, create nodes and xpaths
						if self.parse_html:
									if html_strings is None:
												raise ValueError("""Make sure to pass HTML strings in case `parse_html` is set to `True`"""			)

									if nodes is not None or xpaths is not None or node_labels is not None:
												raise ValueError(
												    """Please don't pass nodes, xpaths nor node labels in case `parse_html` is set to `True`"""			)

									lowerCAmelCase_       :Union[str, Any]     	=    self.feature_extractor(__A			)
									lowerCAmelCase_       :Any     	=    features["""nodes"""]
									lowerCAmelCase_       :Union[str, Any]     	=    features["""xpaths"""]
						else:
									if html_strings is not None:
												raise ValueError("""You have passed HTML strings but `parse_html` is set to `False`."""			)
									if nodes is None or xpaths is None:
												raise ValueError("""Make sure to pass nodes and xpaths in case `parse_html` is set to `False`"""			)

        # # second, apply the tokenizer
						if questions is not None and self.parse_html:
									if isinstance(__A			,  __A			):
												lowerCAmelCase_       :Dict     	=    [questions]  # add batch dimension (as the feature extractor always adds a batch dimension)

						lowerCAmelCase_       :Optional[Any]     	=    self.tokenizer(
						    text=questions if questions is not None else nodes			,  text_pair=nodes if questions is not None else None			,  xpaths=__A			,  node_labels=__A			,  add_special_tokens=__A			,  padding=__A			,  truncation=__A			,  max_length=__A			,  stride=__A			,  pad_to_multiple_of=__A			,  return_token_type_ids=__A			,  return_attention_mask=__A			,  return_overflowing_tokens=__A			,  return_special_tokens_mask=__A			,  return_offsets_mapping=__A			,  return_length=__A			,  verbose=__A			,  return_tensors=__A			,  **__A			,  )

						return encoded_inputs






			def 					__lowerCAmelCase					(   self			,  *__A			,  **__A			)     ->	Tuple:
						return self.tokenizer.batch_decode(*__A			,  **__A			)






			def 					__lowerCAmelCase					(   self			,  *__A			,  **__A			)     ->	str:
						return self.tokenizer.decode(*__A			,  **__A			)






			@property
			def 					__lowerCAmelCase					(   self			)     ->	Any:
						lowerCAmelCase_       :Any     	=    self.tokenizer.model_input_names
						return tokenizer_input_names






