"""simple docstring"""




import unittest

import numpy as np
import torch

from .utils_summarization import build_mask, compute_token_type_ids, process_story, truncate_or_pad
class 						__A							( unittest.TestCase		):






     def     __A  (		self  ):
       _lowerCAmelCase						:       int  			=     10






     def     __A  (		self  ):
       _lowerCAmelCase						:       str  			=     [1, 2, 3, 4]
       _lowerCAmelCase						:       int  			=     [1, 2, 3, 4, 0, 0, 0, 0, 0, 0]
       self.assertEqual(truncate_or_pad(a__				,     self.block_size				,     0  )				,     a__  )






     def     __A  (		self  ):
       _lowerCAmelCase						:       int  			=     [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
       _lowerCAmelCase						:       Any  			=     [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
       self.assertEqual(truncate_or_pad(a__				,     self.block_size				,     0  )				,     a__  )






     def     __A  (		self  ):
       _lowerCAmelCase						:       str  			=     [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
       _lowerCAmelCase						:       Optional[Any]  			=     [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
       self.assertEqual(truncate_or_pad(a__				,     self.block_size				,     0  )				,     a__  )






     def     __A  (		self  ):
       _lowerCAmelCase						:       Union[str, Any]  			=     """It was the year of Our Lord one thousand seven hundred and
        seventy-five.\n\nSpiritual revelations were conceded to England at that
        favoured period, as at this."""
       _lowerCAmelCase			,	_lowerCAmelCase						:       Optional[int]  			=     process_story(a__  )
       self.assertEqual(a__				,     []  )






     def     __A  (		self  ):
       _lowerCAmelCase						:       List[Any]  			=     """"""
       _lowerCAmelCase			,	_lowerCAmelCase						:       Union[str, Any]  			=     process_story(a__  )
       self.assertEqual(a__				,     []  )
       self.assertEqual(a__				,     []  )






     def     __A  (		self  ):
       _lowerCAmelCase						:       str  			=     (
           """It was the year of Our Lord one thousand seven hundred and """
           """seventy-five\n\nSpiritual revelations were conceded to England """
           """at that favoured period, as at this.\n@highlight\n\nIt was the best of times"""
       )
       _lowerCAmelCase			,	_lowerCAmelCase						:       List[str]  			=     process_story(a__  )

       _lowerCAmelCase						:       Union[str, Any]  			=     [
           """It was the year of Our Lord one thousand seven hundred and seventy-five.""",
           """Spiritual revelations were conceded to England at that favoured period, as at this.""",
       ]
       self.assertEqual(a__				,     a__  )

       _lowerCAmelCase						:       List[str]  			=     ["""It was the best of times."""]
       self.assertEqual(a__				,     a__  )






     def     __A  (		self  ):
       _lowerCAmelCase						:       Any  			=     torch.tensor([1, 2, 3, 4]  )
       _lowerCAmelCase						:       Union[str, Any]  			=     torch.tensor([1, 1, 1, 1]  )
       np.testing.assert_array_equal(build_mask(a__				,     0  ).numpy()				,     expected.numpy()  )






     def     __A  (		self  ):
       _lowerCAmelCase						:       Tuple  			=     torch.tensor([1, 2, 3, 4, 23, 23, 23]  )
       _lowerCAmelCase						:       Any  			=     torch.tensor([1, 1, 1, 1, 0, 0, 0]  )
       np.testing.assert_array_equal(build_mask(a__				,     23  ).numpy()				,     expected.numpy()  )






     def     __A  (		self  ):
       _lowerCAmelCase						:       Tuple  			=     torch.tensor([8, 2, 3, 4, 1, 1, 1]  )
       _lowerCAmelCase						:       Dict  			=     torch.tensor([1, 1, 1, 1, 0, 0, 0]  )
       np.testing.assert_array_equal(build_mask(a__				,     1  ).numpy()				,     expected.numpy()  )






     def     __A  (		self  ):
       _lowerCAmelCase						:       int  			=     101
       _lowerCAmelCase						:       Tuple  			=     torch.tensor([[1, 2, 3, 4, 5, 6], [1, 2, 3, 101, 5, 6], [1, 101, 3, 4, 101, 6]]  )
       _lowerCAmelCase						:       Dict  			=     torch.tensor([[1, 1, 1, 1, 1, 1], [1, 1, 1, 0, 0, 0], [1, 0, 0, 0, 1, 1]]  )

       _lowerCAmelCase						:       int  			=     compute_token_type_ids(a__				,     a__  )
       np.testing.assert_array_equal(a__				,     a__  )





"""simple docstring"""




import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


_a      :		Dict           =   datasets.utils.logging.get_logger(__name__)
@dataclass
class 						__A							( datasets.BuilderConfig		):
     _UpperCamelCase							:			int							=		10_000
     _UpperCamelCase							:			Optional[List[str]]							=		None
     _UpperCamelCase							:			Optional[datasets.Features]							=		None







class 						__A							( datasets.ArrowBasedBuilder		):
     _UpperCamelCase							:			List[str]							=		ParquetConfig






     def     __A  (		self  ):
       return datasets.DatasetInfo(features=self.config.features  )






     def     __A  (		self				,     a__  ):
       if not self.config.data_files:
         raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}"  )
       _lowerCAmelCase						:       Optional[Any]  			=     dl_manager.download_and_extract(self.config.data_files  )
       if isinstance(a__				,     (str, list, tuple)  ):
         _lowerCAmelCase						:       Any  			=     data_files
         if isinstance(a__				,     a__  ):
           _lowerCAmelCase						:       Tuple  			=     [files]
         # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
         _lowerCAmelCase						:       Any  			=     [dl_manager.iter_files(a__  ) for file in files]
         return [datasets.SplitGenerator(name=datasets.Split.TRAIN				,     gen_kwargs={"""files""": files}  )]
       _lowerCAmelCase						:       Optional[Any]  			=     []
       for split_name, files in data_files.items():
         if isinstance(a__				,     a__  ):
           _lowerCAmelCase						:       Dict  			=     [files]
         # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
         _lowerCAmelCase						:       Tuple  			=     [dl_manager.iter_files(a__  ) for file in files]
         # Infer features is they are stoed in the arrow schema
         if self.info.features is None:
           for file in itertools.chain.from_iterable(a__  ):
             with open(a__				,     """rb"""  ) as f:
               _lowerCAmelCase						:       Optional[Any]  			=     datasets.Features.from_arrow_schema(pq.read_schema(a__  )  )
             break
         splits.append(datasets.SplitGenerator(name=a__				,     gen_kwargs={"""files""": files}  )  )
       return splits






     def     __A  (		self				,     a__  ):
       if self.info.features is not None:
         # more expensive cast to support nested features with keys in a different order
         # allows str <-> int/float or str to Audio for example
         _lowerCAmelCase						:       Optional[int]  			=     table_cast(a__				,     self.info.features.arrow_schema  )
       return pa_table






     def     __A  (		self				,     a__  ):
       _lowerCAmelCase						:       Optional[int]  			=     self.info.features.arrow_schema if self.info.features is not None else None
       if self.info.features is not None and self.config.columns is not None:
         if sorted(field.name for field in schema  ) != sorted(self.config.columns  ):
           raise ValueError(
               F"Tried to load parquet data with columns '{self.config.columns}' with mismatching features '{self.info.features}'"  )
       for file_idx, file in enumerate(itertools.chain.from_iterable(a__  )  ):
         with open(a__				,     """rb"""  ) as f:
           _lowerCAmelCase						:       Tuple  			=     pq.ParquetFile(a__  )
           try:
             for batch_idx, record_batch in enumerate(
                 parquet_file.iter_batches(batch_size=self.config.batch_size				,     columns=self.config.columns  )  ):
               _lowerCAmelCase						:       Any  			=     pa.Table.from_batches([record_batch]  )
               # Uncomment for debugging (will print the Arrow table size and elements)
               # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
               # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
               yield F"{file_idx}_{batch_idx}", self._cast_table(a__  )
           except ValueError as e:
             logger.error(F"Failed to read file '{file}' with error {type(a__  )}: {e}"  )
             raise




