





"""simple docstring"""




import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


lowerCamelCase_	:		Any         =   datasets.utils.logging.get_logger(__name__)

@dataclass
class 					_UpperCAmelCase    (  datasets.BuilderConfig						):



      '''simple docstring'''
      lowercase_ :					Union[str, Any]             =  None


class 					_UpperCAmelCase    (  datasets.ArrowBasedBuilder						):



      '''simple docstring'''
      lowercase_ :					Any             =  ArrowConfig

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        return datasets.DatasetInfo(features=self.config.features    )

      def    lowerCamelCase_     (	self   , snake_case_    ):

        """simple docstring"""

        if not self.config.data_files:
          raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""    )
        A_    :			int									=    dl_manager.download_and_extract(self.config.data_files    )
        if isinstance(_lowerCamelCase   , (str, list, tuple)    ):
          A_    :			List[str]									=    data_files
          if isinstance(_lowerCamelCase   , _lowerCamelCase    ):
            A_    :			Optional[Any]									=    [files]
          # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
          A_    :			Optional[Any]									=    [dl_manager.iter_files(_lowerCamelCase    ) for file in files]
          return [datasets.SplitGenerator(name=datasets.Split.TRAIN   , gen_kwargs={'files': files}    )]
        A_    :			List[str]									=    []
        for split_name, files in data_files.items():
          if isinstance(_lowerCamelCase   , _lowerCamelCase    ):
            A_    :			Optional[Any]									=    [files]
          # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
          A_    :			Optional[int]									=    [dl_manager.iter_files(_lowerCamelCase    ) for file in files]
          # Infer features is they are stoed in the arrow schema
          if self.info.features is None:
            for file in itertools.chain.from_iterable(_lowerCamelCase    ):
              with open(_lowerCamelCase   , 'rb'    ) as f:
                A_    :			Any									=    datasets.Features.from_arrow_schema(pa.ipc.open_stream(_lowerCamelCase    ).schema    )
              break
          splits.append(datasets.SplitGenerator(name=_lowerCamelCase   , gen_kwargs={'files': files}    )    )
        return splits

      def    lowerCamelCase_     (	self   , snake_case_    ):

        """simple docstring"""

        if self.info.features is not None:
          # more expensive cast to support nested features with keys in a different order
          # allows str <-> int/float or str to Audio for example
          A_    :			int									=    table_cast(_lowerCamelCase   , self.info.features.arrow_schema    )
        return pa_table

      def    lowerCamelCase_     (	self   , snake_case_    ):

        """simple docstring"""

        for file_idx, file in enumerate(itertools.chain.from_iterable(_lowerCamelCase    )    ):
          with open(_lowerCamelCase   , 'rb'    ) as f:
            try:
              for batch_idx, record_batch in enumerate(pa.ipc.open_stream(_lowerCamelCase    )    ):
                A_    :			Tuple									=    pa.Table.from_batches([record_batch]    )
                # Uncomment for debugging (will print the Arrow table size and elements)
                # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                yield F"""{file_idx}_{batch_idx}""", self._cast_table(_lowerCamelCase    )
            except ValueError as e:
              logger.error(F"""Failed to read file '{file}' with error {type(_lowerCamelCase    )}: {e}"""    )
              raise




'''simple docstring'''





def     UpperCAmelCase     (    a_      ,		a_		)   ->       set:



   """simple docstring"""






   assert isinstance(a_      ,		a_		), F"The input value of [set_a={set_a}] is not a set"
   assert isinstance(a_      ,		a_		), F"The input value of [set_b={set_b}] is not a set"

   return {a + b for a in set_a for b in set_b}


if __name__ == "__main__":
  from doctest import testmod

  testmod()
