
import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


_UpperCAmelCase    			=					datasets.utils.logging.get_logger(__name__)

@dataclass
class 		__magic_name__				( datasets.BuilderConfig					):


  """simple docstring"""





  _UpperCamelCase						 = None







class 		__magic_name__				( datasets.ArrowBasedBuilder					):


  """simple docstring"""





  _UpperCamelCase						 = ArrowConfig


  def     _UpperCAmelCase  (					self		):
    return datasets.DatasetInfo(features=self.config.features		)


  def     _UpperCAmelCase  (					self	,		a__		):
    if not self.config.data_files:
      raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}'''		)
    _lowerCamelCase      =						dl_manager.download_and_extract(self.config.data_files		)
    if isinstance(a__	,		(str, list, tuple)		):
      _lowerCamelCase      =						data_files
      if isinstance(a__	,		a__		):
        _lowerCamelCase      =						[files]
      # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
      _lowerCamelCase      =						[dl_manager.iter_files(a__		) for file in files]
      return [datasets.SplitGenerator(name=datasets.Split.TRAIN	,		gen_kwargs={'''files''': files}		)]
    _lowerCamelCase      =						[]
    for split_name, files in data_files.items():
      if isinstance(a__	,		a__		):
        _lowerCamelCase      =						[files]
      # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
      _lowerCamelCase      =						[dl_manager.iter_files(a__		) for file in files]
      # Infer features is they are stoed in the arrow schema
      if self.info.features is None:
        for file in itertools.chain.from_iterable(a__		):
          with open(a__	,		'''rb'''		) as f:
            _lowerCamelCase      =						datasets.Features.from_arrow_schema(pa.ipc.open_stream(a__		).schema		)
          break
      splits.append(datasets.SplitGenerator(name=a__	,		gen_kwargs={'''files''': files}		)		)
    return splits


  def     _UpperCAmelCase  (					self	,		a__		):
    if self.info.features is not None:
      # more expensive cast to support nested features with keys in a different order
      # allows str <-> int/float or str to Audio for example
      _lowerCamelCase      =						table_cast(a__	,		self.info.features.arrow_schema		)
    return pa_table


  def     _UpperCAmelCase  (					self	,		a__		):
    for file_idx, file in enumerate(itertools.chain.from_iterable(a__		)		):
      with open(a__	,		'''rb'''		) as f:
        try:
          for batch_idx, record_batch in enumerate(pa.ipc.open_stream(a__		)		):
            _lowerCamelCase      =						pa.Table.from_batches([record_batch]		)
            # Uncomment for debugging (will print the Arrow table size and elements)
            # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
            # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
            yield f'''{file_idx}_{batch_idx}''', self._cast_table(a__		)
        except ValueError as e:
          logger.error(f'''Failed to read file \'{file}\' with error {type(a__		)}: {e}'''		)
          raise






import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin

class 		__magic_name__				( lowercase_					):


  """simple docstring"""





  _UpperCamelCase						 = ["image_processor", "tokenizer"]
  _UpperCamelCase						 = "AutoImageProcessor"
  _UpperCamelCase						 = "AutoTokenizer"


  def __init__(					self	,		a__=None	,		a__=None	,		**a__		):
    _lowerCamelCase      =						None
    if "feature_extractor" in kwargs:
      warnings.warn(
          '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
          ''' instead.'''	,		a__	,		)
      _lowerCamelCase      =						kwargs.pop('''feature_extractor'''		)

    _lowerCamelCase      =						image_processor if image_processor is not None else feature_extractor
    if image_processor is None:
      raise ValueError('''You need to specify an `image_processor`.'''		)
    if tokenizer is None:
      raise ValueError('''You need to specify a `tokenizer`.'''		)

    super().__init__(a__	,		a__		)
    _lowerCamelCase      =						self.image_processor
    _lowerCamelCase      =						False


  def __call__(					self	,		*a__	,		**a__		):
    # For backward compatibility
    if self._in_target_context_manager:
      return self.current_processor(*a__	,		**a__		)

    _lowerCamelCase      =						kwargs.pop('''images'''	,		a__		)
    _lowerCamelCase      =						kwargs.pop('''text'''	,		a__		)
    if len(a__		) > 0:
      _lowerCamelCase      =						args[0]
      _lowerCamelCase      =						args[1:]

    if images is None and text is None:
      raise ValueError('''You need to specify either an `images` or `text` input to process.'''		)

    if images is not None:
      _lowerCamelCase      =						self.image_processor(a__	,		*a__	,		**a__		)
    if text is not None:
      _lowerCamelCase      =						self.tokenizer(a__	,		**a__		)

    if text is None:
      return inputs
    elif images is None:
      return encodings
    else:
      _lowerCamelCase      =						encodings['''input_ids''']
      return inputs


  def     _UpperCAmelCase  (					self	,		*a__	,		**a__		):
    return self.tokenizer.batch_decode(*a__	,		**a__		)


  def     _UpperCAmelCase  (					self	,		*a__	,		**a__		):
    return self.tokenizer.decode(*a__	,		**a__		)


  @contextmanager
  def     _UpperCAmelCase  (					self		):
    warnings.warn(
        '''`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '''
        '''labels by using the argument `text` of the regular `__call__` method (either in the same call as '''
        '''your images inputs, or in a separate call.'''		)
    _lowerCamelCase      =						True
    _lowerCamelCase      =						self.tokenizer
    yield
    _lowerCamelCase      =						self.image_processor
    _lowerCamelCase      =						False


  @property
  def     _UpperCAmelCase  (					self		):
    warnings.warn(
        '''`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'''	,		a__	,		)
    return self.image_processor_class


  @property
  def     _UpperCAmelCase  (					self		):
    warnings.warn(
        '''`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'''	,		a__	,		)
    return self.image_processor




