

import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


_a		       =   datasets.utils.logging.get_logger(__name__)






@dataclass
class 							A_ (		datasets.BuilderConfig					):
     _lowercase			:     Optional[Any]								=	1_0_0_0_0
     _lowercase			:     Optional[Any]								=	None
     _lowercase			:     Union[str, Any]								=	None







class 							A_ (		datasets.ArrowBasedBuilder					):
     _lowercase			:     str								=	ParquetConfig


     def   UpperCAmelCase  (							self			:		Optional[Any]					)							->	Union[str, Any]:
      return datasets.DatasetInfo(features=self.config.features					)


     def   UpperCAmelCase  (							self			:		List[str]			,  UpperCAmelCase			:		str					)							->	List[str]:
      if not self.config.data_files:
       raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''					)
      __lowerCAmelCase:   Dict   	=      dl_manager.download_and_extract(self.config.data_files					)
      if isinstance(UpperCAmelCase			,  (str, list, tuple)					):
       __lowerCAmelCase:   Dict   	=      data_files
       if isinstance(UpperCAmelCase			,  UpperCAmelCase					):
        __lowerCAmelCase:   Optional[int]   	=      [files]
       # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
       __lowerCAmelCase:   int   	=      [dl_manager.iter_files(UpperCAmelCase					) for file in files]
       return [datasets.SplitGenerator(name=datasets.Split.TRAIN			,  gen_kwargs={'files': files}					)]
      __lowerCAmelCase:   int   	=      []
      for split_name, files in data_files.items():
       if isinstance(UpperCAmelCase			,  UpperCAmelCase					):
        __lowerCAmelCase:   str   	=      [files]
       # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
       __lowerCAmelCase:   List[str]   	=      [dl_manager.iter_files(UpperCAmelCase					) for file in files]
       # Infer features is they are stoed in the arrow schema
       if self.info.features is None:
        for file in itertools.chain.from_iterable(UpperCAmelCase					):
         with open(UpperCAmelCase			,  'rb'					) as f:
          __lowerCAmelCase:   Union[str, Any]   	=      datasets.Features.from_arrow_schema(pq.read_schema(UpperCAmelCase					)					)
         break
       splits.append(datasets.SplitGenerator(name=UpperCAmelCase			,  gen_kwargs={'files': files}					)					)
      return splits


     def   UpperCAmelCase  (							self			:		Optional[int]			,  UpperCAmelCase			:		Dict					)							->	pa.Table:
      if self.info.features is not None:
       # more expensive cast to support nested features with keys in a different order
       # allows str <-> int/float or str to Audio for example
       __lowerCAmelCase:   List[Any]   	=      table_cast(UpperCAmelCase			,  self.info.features.arrow_schema					)
      return pa_table


     def   UpperCAmelCase  (							self			:		Tuple			,  UpperCAmelCase			:		Any					)							->	Tuple:
      __lowerCAmelCase:   Union[str, Any]   	=      self.info.features.arrow_schema if self.info.features is not None else None
      if self.info.features is not None and self.config.columns is not None:
       if sorted(field.name for field in schema					) != sorted(self.config.columns					):
        raise ValueError(
            F'''Tried to load parquet data with columns \'{self.config.columns}\' with mismatching features \'{self.info.features}\''''					)
      for file_idx, file in enumerate(itertools.chain.from_iterable(UpperCAmelCase					)					):
       with open(UpperCAmelCase			,  'rb'					) as f:
        __lowerCAmelCase:   Optional[Any]   	=      pq.ParquetFile(UpperCAmelCase					)
        try:
         for batch_idx, record_batch in enumerate(
             parquet_file.iter_batches(batch_size=self.config.batch_size			,  columns=self.config.columns					)					):
          __lowerCAmelCase:   Dict   	=      pa.Table.from_batches([record_batch]					)
          # Uncomment for debugging (will print the Arrow table size and elements)
          # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
          # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
          yield F'''{file_idx}_{batch_idx}''', self._cast_table(UpperCAmelCase					)
        except ValueError as e:
         logger.error(F'''Failed to read file \'{file}\' with error {type(UpperCAmelCase					)}: {e}'''					)
         raise









from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


UpperCAmelCase__   	=							{
    '''configuration_xlm''': ['''XLM_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''XLMConfig''', '''XLMOnnxConfig'''],
    '''tokenization_xlm''': ['''XLMTokenizer'''],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase__   	=							[
      '''XLM_PRETRAINED_MODEL_ARCHIVE_LIST''',
      '''XLMForMultipleChoice''',
      '''XLMForQuestionAnswering''',
      '''XLMForQuestionAnsweringSimple''',
      '''XLMForSequenceClassification''',
      '''XLMForTokenClassification''',
      '''XLMModel''',
      '''XLMPreTrainedModel''',
      '''XLMWithLMHeadModel''',
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase__   	=							[
      '''TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST''',
      '''TFXLMForMultipleChoice''',
      '''TFXLMForQuestionAnsweringSimple''',
      '''TFXLMForSequenceClassification''',
      '''TFXLMForTokenClassification''',
      '''TFXLMMainLayer''',
      '''TFXLMModel''',
      '''TFXLMPreTrainedModel''',
      '''TFXLMWithLMHeadModel''',
  ]


if TYPE_CHECKING:
  from .configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig, XLMOnnxConfig
  from .tokenization_xlm import XLMTokenizer

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_xlm import (
        XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
        XLMForMultipleChoice,
        XLMForQuestionAnswering,
        XLMForQuestionAnsweringSimple,
        XLMForSequenceClassification,
        XLMForTokenClassification,
        XLMModel,
        XLMPreTrainedModel,
        XLMWithLMHeadModel,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_xlm import (
        TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFXLMForMultipleChoice,
        TFXLMForQuestionAnsweringSimple,
        TFXLMForSequenceClassification,
        TFXLMForTokenClassification,
        TFXLMMainLayer,
        TFXLMModel,
        TFXLMPreTrainedModel,
        TFXLMWithLMHeadModel,
    )

else:
  import sys

  UpperCAmelCase__   	=							_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)

