





"""simple docstring"""


import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


a__       :  Any       		=					datasets.utils.logging.get_logger(__name__)



@dataclass
class   lowercase_					(  datasets.BuilderConfig				):
       snake_case__					:				int									=  None
       snake_case__					:				str									=  """utf-8"""
       snake_case__					:				int									=  None
       snake_case__					:				Optional[Any]									=  10 << 20  # 10MB
       snake_case__					:				List[Any]									=  False
       snake_case__					:				str									=  """line"""



class   lowercase_					(  datasets.ArrowBasedBuilder				):
       snake_case__					:				List[Any]									=  TextConfig

       def 		_SCREAMING_SNAKE_CASE(      self  ):






        '''simple docstring'''

        return datasets.DatasetInfo(features=self.config.features  )

       def 		_SCREAMING_SNAKE_CASE(      self ,       __SCREAMING_SNAKE_CASE  ):






        '''simple docstring'''

        if not self.config.data_files:
         raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}"  )
        snake_case__     				=					dl_manager.download_and_extract(self.config.data_files  )
        if isinstance(snake_case__ ,       (str, list, tuple)  ):
         snake_case__     				=					data_files
         if isinstance(snake_case__ ,       snake_case__  ):
          snake_case__     				=					[files]
         snake_case__     				=					[dl_manager.iter_files(snake_case__  ) for file in files]
         return [datasets.SplitGenerator(name=datasets.Split.TRAIN ,       gen_kwargs={"files": files}  )]
        snake_case__     				=					[]
        for split_name, files in data_files.items():
         if isinstance(snake_case__ ,       snake_case__  ):
          snake_case__     				=					[files]
         snake_case__     				=					[dl_manager.iter_files(snake_case__  ) for file in files]
         splits.append(datasets.SplitGenerator(name=snake_case__ ,       gen_kwargs={"files": files}  )  )
        return splits

       def 		_SCREAMING_SNAKE_CASE(      self ,       __SCREAMING_SNAKE_CASE  ):






        '''simple docstring'''

        if self.config.features is not None:
         snake_case__     				=					self.config.features.arrow_schema
         if all(not require_storage_cast(snake_case__  ) for feature in self.config.features.values()  ):
          # cheaper cast
          snake_case__     				=					pa_table.cast(snake_case__  )
         else:
          # more expensive cast; allows str <-> int/float or str to Audio for example
          snake_case__     				=					table_cast(snake_case__ ,       snake_case__  )
         return pa_table
        else:
         return pa_table.cast(pa.schema({"text": pa.string()}  )  )

       def 		_SCREAMING_SNAKE_CASE(      self ,       __SCREAMING_SNAKE_CASE  ):






        '''simple docstring'''

        snake_case__     				=					list(self.config.features  ) if self.config.features is not None else ["text"]
        for file_idx, file in enumerate(itertools.chain.from_iterable(snake_case__  )  ):
         # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
         with open(snake_case__ ,       encoding=self.config.encoding ,       errors=self.config.errors  ) as f:
          if self.config.sample_by == "line":
           snake_case__     				=					0
           while True:
            snake_case__     				=					f.read(self.config.chunksize  )
            if not batch:
             break
            batch += f.readline()  # finish current line
            # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
            snake_case__     				=					StringIO(snake_case__  ).readlines()
            if not self.config.keep_linebreaks:
             snake_case__     				=					[line.rstrip("\n"  ) for line in batch]
            snake_case__     				=					pa.Table.from_arrays([pa.array(snake_case__  )] ,       names=snake_case__  )
            # Uncomment for debugging (will print the Arrow table size and elements)
            # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
            # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
            yield (file_idx, batch_idx), self._cast_table(snake_case__  )
            batch_idx += 1
          elif self.config.sample_by == "paragraph":
           snake_case__     				=					0
           snake_case__     				=					""
           while True:
            snake_case__     				=					f.read(self.config.chunksize  )
            if not new_batch:
             break
            batch += new_batch
            batch += f.readline()  # finish current line
            snake_case__     				=					batch.split("\n\n"  )
            snake_case__     				=					pa.Table.from_arrays(
                [pa.array([example for example in batch[:-1] if example]  )] ,       names=snake_case__  )
            # Uncomment for debugging (will print the Arrow table size and elements)
            # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
            # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
            yield (file_idx, batch_idx), self._cast_table(snake_case__  )
            batch_idx += 1
            snake_case__     				=					batch[-1]
           if batch:
            snake_case__     				=					pa.Table.from_arrays([pa.array([batch]  )] ,       names=snake_case__  )
            yield (file_idx, batch_idx), self._cast_table(snake_case__  )
          elif self.config.sample_by == "document":
           snake_case__     				=					f.read()
           snake_case__     				=					pa.Table.from_arrays([pa.array([text]  )] ,       names=snake_case__  )
           yield file_idx, self._cast_table(snake_case__  )









from __future__ import annotations

from enum import Enum, unique
from typing import TypeVar

# Create a generic variable that can be 'Enum', or any subclass.
lowerCamelCase					    =	TypeVar('''T''', bound='''Enum''')

@unique
class        snake_case_				(							lowercase_   ):







   """simple docstring"""



   __lowerCAmelCase  =    80
   __lowerCAmelCase  =    70
   __lowerCAmelCase  =    60
   __lowerCAmelCase  =    50
   __lowerCAmelCase  =    40
   __lowerCAmelCase  =    30
   __lowerCAmelCase  =    20
   __lowerCAmelCase  =    10






@unique
class        snake_case_				(							lowercase_   ):







   """simple docstring"""



   __lowerCAmelCase  =    24
   __lowerCAmelCase  =    21
   __lowerCAmelCase  =    18
   __lowerCAmelCase  =    15
   __lowerCAmelCase  =    12
   __lowerCAmelCase  =    9
   __lowerCAmelCase  =    6
   __lowerCAmelCase  =    3
   __lowerCAmelCase  =    2
   __lowerCAmelCase  =    1
   __lowerCAmelCase  =    -1
   __lowerCAmelCase  =    -2
   __lowerCAmelCase  =    -3
   __lowerCAmelCase  =    -6
   __lowerCAmelCase  =    -9
   __lowerCAmelCase  =    -12
   __lowerCAmelCase  =    -15
   __lowerCAmelCase  =    -18
   __lowerCAmelCase  =    -21
   __lowerCAmelCase  =    -24

   @classmethod
   def       SCREAMING_SNAKE_CASE__					(     cls:						type[T]						)	->		dict:


       """simple docstring"""



       return {unit.name: unit.value for unit in cls if unit.value > 0}

   @classmethod
   def       SCREAMING_SNAKE_CASE__					(     cls:						type[T]						)	->		dict:


       """simple docstring"""



       return {unit.name: unit.value for unit in cls if unit.value < 0}






def 							UpperCAmelCase   ( _lowercase   ):


    '''simple docstring'''



    _lowercase     			=						SIUnit.get_positive() if value > 0 else SIUnit.get_negative()
    for name_prefix, value_prefix in prefixes.items():
        _lowercase     			=						value / (10**value_prefix)
        if numerical_part > 1:
            return f'''{numerical_part!s} {name_prefix}'''
    return str(_lowercase   )
def 							UpperCAmelCase   ( _lowercase   ):


    '''simple docstring'''



    for prefix in BinaryUnit:
        _lowercase     			=						value / (2**prefix.value)
        if numerical_part > 1:
            return f'''{numerical_part!s} {prefix.name}'''
    return str(_lowercase   )


if __name__ == "__main__":
   import doctest

   doctest.testmod()
