


'''simple docstring'''



import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


__UpperCAmelCase     :List[str]												=  datasets.utils.logging.get_logger(__name__)



@dataclass
class       a							(						datasets.BuilderConfig			):


     """simple docstring"""






     SCREAMING_SNAKE_CASE			:					Optional[datasets.Features]				  =  None
     SCREAMING_SNAKE_CASE			:					str				  =  "utf-8"
     SCREAMING_SNAKE_CASE			:					Optional[str]				  =  None
     SCREAMING_SNAKE_CASE			:					int				  =  1_0 << 2_0  # 10MB
     SCREAMING_SNAKE_CASE			:					bool				  =  False
     SCREAMING_SNAKE_CASE			:					str				  =  "line"



class       a							(						datasets.ArrowBasedBuilder			):


     """simple docstring"""






     SCREAMING_SNAKE_CASE			:					Any				  =  TextConfig

     def 					lowerCamelCase__					(							self	:						int     )		->				Any:
          return datasets.DatasetInfo(features=self.config.features     )

     def 					lowerCamelCase__					(							self	:						str				,					snake_case	:						List[Any]     )		->				Optional[Any]:
          if not self.config.data_files:
               raise ValueError(f'At least one data file must be specified, but got data_files={self.config.data_files}'     )
          __UpperCAmelCase      :					Tuple    				=     dl_manager.download_and_extract(self.config.data_files     )
          if isinstance(snake_case				,					(str, list, tuple)     ):
               __UpperCAmelCase      :					Union[str, Any]    				=     data_files
               if isinstance(snake_case				,					snake_case     ):
                    __UpperCAmelCase      :					str    				=     [files]
               __UpperCAmelCase      :					str    				=     [dl_manager.iter_files(snake_case     ) for file in files]
               return [datasets.SplitGenerator(name=datasets.Split.TRAIN				,					gen_kwargs={'''files''': files}     )]
          __UpperCAmelCase      :					List[Any]    				=     []
          for split_name, files in data_files.items():
               if isinstance(snake_case				,					snake_case     ):
                    __UpperCAmelCase      :					Union[str, Any]    				=     [files]
               __UpperCAmelCase      :					str    				=     [dl_manager.iter_files(snake_case     ) for file in files]
               splits.append(datasets.SplitGenerator(name=snake_case				,					gen_kwargs={'''files''': files}     )     )
          return splits

     def 					lowerCamelCase__					(							self	:						Optional[Any]				,					snake_case	:						pa.Table     )		->				pa.Table:
          if self.config.features is not None:
               __UpperCAmelCase      :					Optional[Any]    				=     self.config.features.arrow_schema
               if all(not require_storage_cast(snake_case     ) for feature in self.config.features.values()     ):
                    # cheaper cast
                    __UpperCAmelCase      :					Tuple    				=     pa_table.cast(snake_case     )
               else:
                    # more expensive cast; allows str <-> int/float or str to Audio for example
                    __UpperCAmelCase      :					Dict    				=     table_cast(snake_case				,					snake_case     )
               return pa_table
          else:
               return pa_table.cast(pa.schema({'''text''': pa.string()}     )     )






     def 					lowerCamelCase__					(							self	:						str				,					snake_case	:						Union[str, Any]     )		->				List[str]:
          __UpperCAmelCase      :					List[Any]    				=     list(self.config.features     ) if self.config.features is not None else ['''text''']
          for file_idx, file in enumerate(itertools.chain.from_iterable(snake_case     )     ):
               # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
               with open(snake_case				,					encoding=self.config.encoding				,					errors=self.config.errors     ) as f:
                    if self.config.sample_by == "line":
                         __UpperCAmelCase      :					int    				=     0
                         while True:
                              __UpperCAmelCase      :					int    				=     f.read(self.config.chunksize     )
                              if not batch:
                                   break
                              batch += f.readline()  # finish current line
                              # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
                              __UpperCAmelCase      :					List[str]    				=     StringIO(snake_case     ).readlines()
                              if not self.config.keep_linebreaks:
                                   __UpperCAmelCase      :					List[Any]    				=     [line.rstrip('''\n'''     ) for line in batch]
                              __UpperCAmelCase      :					Optional[int]    				=     pa.Table.from_arrays([pa.array(snake_case     )]				,					names=snake_case     )
                              # Uncomment for debugging (will print the Arrow table size and elements)
                              # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                              # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                              yield (file_idx, batch_idx), self._cast_table(snake_case     )
                              batch_idx += 1
                    elif self.config.sample_by == "paragraph":
                         __UpperCAmelCase      :					Tuple    				=     0
                         __UpperCAmelCase      :					Dict    				=     ''''''
                         while True:
                              __UpperCAmelCase      :					Union[str, Any]    				=     f.read(self.config.chunksize     )
                              if not new_batch:
                                   break
                              batch += new_batch
                              batch += f.readline()  # finish current line
                              __UpperCAmelCase      :					Any    				=     batch.split('''\n\n'''     )
                              __UpperCAmelCase      :					Tuple    				=     pa.Table.from_arrays(
                                  [pa.array([example for example in batch[:-1] if example]     )]				,					names=snake_case     )
                              # Uncomment for debugging (will print the Arrow table size and elements)
                              # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                              # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                              yield (file_idx, batch_idx), self._cast_table(snake_case     )
                              batch_idx += 1
                              __UpperCAmelCase      :					Tuple    				=     batch[-1]
                         if batch:
                              __UpperCAmelCase      :					Union[str, Any]    				=     pa.Table.from_arrays([pa.array([batch]     )]				,					names=snake_case     )
                              yield (file_idx, batch_idx), self._cast_table(snake_case     )
                    elif self.config.sample_by == "document":
                         __UpperCAmelCase      :					Tuple    				=     f.read()
                         __UpperCAmelCase      :					Union[str, Any]    				=     pa.Table.from_arrays([pa.array([text]     )]				,					names=snake_case     )
                         yield file_idx, self._cast_table(snake_case     )



'''simple docstring'''



def       _a      (		_lowercase   :      str    ):





     '''simple docstring'''






     __UpperCAmelCase      :					Union[str, Any]    				=     5381
     for x in s:
          __UpperCAmelCase      :					Union[str, Any]    				=     ((hash_value << 5) + hash_value) + ord(_lowercase    )
     return hash_value & 0XFFFFFFFF