





import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


SCREAMING_SNAKE_CASE_						   =       datasets.utils.logging.get_logger(__name__)

@dataclass
class 		UpperCamelCase__ (      datasets.BuilderConfig   ):



     '''simple docstring'''

     __snake_case    :	int			=			None
     __snake_case    :	Tuple			=			"utf-8"
     __snake_case    :	List[Any]			=			None
     __snake_case    :	Any			=			None
     __snake_case    :	List[str]			=			True  # deprecated
     __snake_case    :	Union[str, Any]			=			None  # deprecated
     __snake_case    :	Dict			=			10 << 20  # 10MB
     __snake_case    :	Optional[int]			=			None

class 		UpperCamelCase__ (      datasets.ArrowBasedBuilder   ):



     '''simple docstring'''

     __snake_case    :	Dict			=			JsonConfig






     def 				SCREAMING_SNAKE_CASE__  (							self						:    str       )     ->		Optional[int]:




           '''simple docstring'''


           if self.config.block_size is not None:
                 logger.warning("""The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"""       )
                 SCREAMING_SNAKE_CASE		=						self.config.block_size
           if self.config.use_threads is not True:
                 logger.warning(
                     """The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."""       )
           if self.config.newlines_in_values is not None:
                 raise ValueError("""The JSON loader parameter `newlines_in_values` is no longer supported"""       )
           return datasets.DatasetInfo(features=self.config.features       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    str						,lowerCamelCase__						:    Dict       )     ->		Union[str, Any]:




           '''simple docstring'''


           if not self.config.data_files:
                 raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""       )
           SCREAMING_SNAKE_CASE		=						dl_manager.download_and_extract(self.config.data_files       )
           if isinstance(lowerCamelCase__						,(str, list, tuple)       ):
                 SCREAMING_SNAKE_CASE		=						data_files
                 if isinstance(lowerCamelCase__						,lowerCamelCase__       ):
                       SCREAMING_SNAKE_CASE		=						[files]
                 SCREAMING_SNAKE_CASE		=						[dl_manager.iter_files(lowerCamelCase__       ) for file in files]
                 return [datasets.SplitGenerator(name=datasets.Split.TRAIN						,gen_kwargs={"""files""": files}       )]
           SCREAMING_SNAKE_CASE		=						[]
           for split_name, files in data_files.items():
                 if isinstance(lowerCamelCase__						,lowerCamelCase__       ):
                       SCREAMING_SNAKE_CASE		=						[files]
                 SCREAMING_SNAKE_CASE		=						[dl_manager.iter_files(lowerCamelCase__       ) for file in files]
                 splits.append(datasets.SplitGenerator(name=lowerCamelCase__						,gen_kwargs={"""files""": files}       )       )
           return splits






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Union[str, Any]						,lowerCamelCase__						:    List[Any]       )     ->		Dict:




           '''simple docstring'''


           if self.config.features is not None:
                 # adding missing columns
                 for column_name in set(self.config.features       ) - set(pa_table.column_names       ):
                       SCREAMING_SNAKE_CASE		=						self.config.features.arrow_schema.field(lowerCamelCase__       ).type
                       SCREAMING_SNAKE_CASE		=						pa_table.append_column(lowerCamelCase__						,pa.array([None] * len(lowerCamelCase__       )						,type=lowerCamelCase__       )       )
                 # more expensive cast to support nested structures with keys in a different order
                 # allows str <-> int/float or str to Audio for example
                 SCREAMING_SNAKE_CASE		=						table_cast(lowerCamelCase__						,self.config.features.arrow_schema       )
           return pa_table






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Optional[int]						,lowerCamelCase__						:    Tuple       )     ->		Optional[Any]:




           '''simple docstring'''


           for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCamelCase__       )       ):
                 # If the file is one json object and if we need to look at the list of items in one specific field
                 if self.config.field is not None:
                       with open(lowerCamelCase__						,encoding=self.config.encoding						,errors=self.config.encoding_errors       ) as f:
                             SCREAMING_SNAKE_CASE		=						json.load(lowerCamelCase__       )

                       # We keep only the field we are interested in
                       SCREAMING_SNAKE_CASE		=						dataset[self.config.field]

                       # We accept two format: a list of dicts or a dict of lists
                       if isinstance(lowerCamelCase__						,(list, tuple)       ):
                             SCREAMING_SNAKE_CASE		=						set().union(*[row.keys() for row in dataset]       )
                             SCREAMING_SNAKE_CASE		=						{col: [row.get(lowerCamelCase__       ) for row in dataset] for col in keys}
                       else:
                             SCREAMING_SNAKE_CASE		=						dataset
                       SCREAMING_SNAKE_CASE		=						pa.Table.from_pydict(lowerCamelCase__       )
                       yield file_idx, self._cast_table(lowerCamelCase__       )

                 # If the file has one json object per line
                 else:
                       with open(lowerCamelCase__						,"""rb"""       ) as f:
                             SCREAMING_SNAKE_CASE		=						0
                             # Use block_size equal to the chunk size divided by 32 to leverage multithreading
                             # Set a default minimum value of 16kB if the chunk size is really small
                             SCREAMING_SNAKE_CASE		=						max(self.config.chunksize // 32						,16 << 10       )
                             SCREAMING_SNAKE_CASE		=						(
                                 self.config.encoding_errors if self.config.encoding_errors is not None else """strict"""
                             )
                             while True:
                                   SCREAMING_SNAKE_CASE		=						f.read(self.config.chunksize       )
                                   if not batch:
                                         break
                                   # Finish current line
                                   try:
                                         batch += f.readline()
                                   except (AttributeError, io.UnsupportedOperation):
                                         batch += readline(lowerCamelCase__       )
                                   # PyArrow only accepts utf-8 encoded bytes
                                   if self.config.encoding != "utf-8":
                                         SCREAMING_SNAKE_CASE		=						batch.decode(self.config.encoding						,errors=lowerCamelCase__       ).encode("""utf-8"""       )
                                   try:
                                         while True:
                                               try:
                                                     SCREAMING_SNAKE_CASE		=						paj.read_json(
                                                         io.BytesIO(lowerCamelCase__       )						,read_options=paj.ReadOptions(block_size=lowerCamelCase__       )       )
                                                     break
                                               except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
                                                     if (
                                                         isinstance(lowerCamelCase__						,pa.ArrowInvalid       )
                                                         and "straddling" not in str(lowerCamelCase__       )
                                                         or block_size > len(lowerCamelCase__       )
                                                     ):
                                                           raise
                                                     else:
                                                           # Increase the block size in case it was too small.
                                                           # The block size will be reset for the next file.
                                                           logger.debug(
                                                               F"""Batch of {len(lowerCamelCase__       )} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."""       )
                                                           block_size *= 2
                                   except pa.ArrowInvalid as e:
                                         try:
                                               with open(
                                                   lowerCamelCase__						,encoding=self.config.encoding						,errors=self.config.encoding_errors       ) as f:
                                                     SCREAMING_SNAKE_CASE		=						json.load(lowerCamelCase__       )
                                         except json.JSONDecodeError:
                                               logger.error(F"""Failed to read file \'{file}\' with error {type(lowerCamelCase__       )}: {e}"""       )
                                               raise e
                                         # If possible, parse the file as a list of json objects and exit the loop
                                         if isinstance(lowerCamelCase__						,lowerCamelCase__       ):  # list is the only sequence type supported in JSON
                                               try:
                                                     SCREAMING_SNAKE_CASE		=						set().union(*[row.keys() for row in dataset]       )
                                                     SCREAMING_SNAKE_CASE		=						{col: [row.get(lowerCamelCase__       ) for row in dataset] for col in keys}
                                                     SCREAMING_SNAKE_CASE		=						pa.Table.from_pydict(lowerCamelCase__       )
                                               except (pa.ArrowInvalid, AttributeError) as e:
                                                     logger.error(F"""Failed to read file \'{file}\' with error {type(lowerCamelCase__       )}: {e}"""       )
                                                     raise ValueError(F"""Not able to read records in the JSON file at {file}."""       ) from None
                                               yield file_idx, self._cast_table(lowerCamelCase__       )
                                               break
                                         else:
                                               logger.error(F"""Failed to read file \'{file}\' with error {type(lowerCamelCase__       )}: {e}"""       )
                                               raise ValueError(
                                                   F"""Not able to read records in the JSON file at {file}. """
                                                   F"""You should probably indicate the field of the JSON file containing your records. """
                                                   F"""This JSON file contain the following fields: {str(list(dataset.keys()       )       )}. """
                                                   F"""Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. """       ) from None
                                                              # Uncomment for debugging (will print the Arrow table size and elements)
                                                              # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                                              # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                   yield (file_idx, batch_idx), self._cast_table(lowerCamelCase__       )
                                   batch_idx += 1






'''simple docstring'''

from typing import List, Optional, Union

import numpy as np
import torch
import torchaudio.compliance.kaldi as ta_kaldi

from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


a_  :				int						= logging.get_logger(__name__)





class      snake_case    (    lowercase			):





  """simple docstring"""




  _lowerCamelCase		    =  ["input_features", "attention_mask"]






  def __init__(    self			,  UpperCamelCase=80			,  UpperCamelCase=1_6000			,  UpperCamelCase=80			,  UpperCamelCase=0.0			,  UpperCamelCase=True			,  UpperCamelCase=True			,  UpperCamelCase=True			,  **UpperCamelCase			,  ):







   """simple docstring"""
   super().__init__(feature_size=UpperCamelCase			,  sampling_rate=UpperCamelCase			,  padding_value=UpperCamelCase			,  **UpperCamelCase     )
   lowerCamelCase_      =  num_mel_bins
   lowerCamelCase_      =  do_ceptral_normalize
   lowerCamelCase_      =  normalize_means
   lowerCamelCase_      =  normalize_vars
   lowerCamelCase_      =  True






  def   snake_case		(    self			,  UpperCamelCase			,  ):







   """simple docstring"""
   lowerCamelCase_      =  waveform * (2**15)  # Kaldi compliance: 16-bit signed integers
   lowerCamelCase_      =  torch.from_numpy(UpperCamelCase     ).unsqueeze(0     )
   lowerCamelCase_      =  ta_kaldi.fbank(UpperCamelCase			,  num_mel_bins=self.num_mel_bins			,  sample_frequency=self.sampling_rate     )
   return features.numpy()






  @staticmethod
  def   snake_case		(    UpperCamelCase			,  UpperCamelCase			,  UpperCamelCase = True			,  UpperCamelCase = True			,  UpperCamelCase = 0.0			,  ):







   """simple docstring"""
   # make sure we normalize float32 arrays
   if normalize_means:
    lowerCamelCase_      =  x[:input_length].mean(axis=0     )
    lowerCamelCase_      =  np.subtract(UpperCamelCase			,  UpperCamelCase     )
   if normalize_vars:
    lowerCamelCase_      =  x[:input_length].std(axis=0     )
    lowerCamelCase_      =  np.divide(UpperCamelCase			,  UpperCamelCase     )

   if input_length < x.shape[0]:
    lowerCamelCase_      =  padding_value

   # make sure array is in float32
   lowerCamelCase_      =  x.astype(np.floataa     )

   return x






  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase = None     ):







   """simple docstring"""
   lowerCamelCase_      =  attention_mask.sum(-1     ) if attention_mask is not None else [x.shape[0] for x in input_features]
   return [
       self.utterance_cmvn(UpperCamelCase			,  UpperCamelCase			,  self.normalize_means			,  self.normalize_vars			,  self.padding_value     )
       for x, n in zip(UpperCamelCase			,  UpperCamelCase     )
   ]







  def __call__(    self			,  UpperCamelCase			,  UpperCamelCase = False			,  UpperCamelCase = None			,  UpperCamelCase = False			,  UpperCamelCase = None			,  UpperCamelCase = None			,  UpperCamelCase = None			,  UpperCamelCase = None			,  **UpperCamelCase			,  ):







   """simple docstring"""

   if sampling_rate is not None:
    if sampling_rate != self.sampling_rate:
     raise ValueError(
         f'''The model corresponding to this feature extractor: {self} was trained using a sampling rate of'''
         f''' {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with'''
         f''' {self.sampling_rate} and not {sampling_rate}.'''     )
   else:
    logger.warning(
        "It is strongly recommended to pass the `sampling_rate` argument to this function. "
        "Failing to do so can result in silent errors that might be hard to debug."     )

   lowerCamelCase_      =  isinstance(UpperCamelCase			,  np.ndarray     ) and len(raw_speech.shape     ) > 1
   if is_batched_numpy and len(raw_speech.shape     ) > 2:
    raise ValueError(f'''Only mono-channel audio is supported for input to {self}'''     )
   lowerCamelCase_      =  is_batched_numpy or (
       isinstance(UpperCamelCase			,  (list, tuple)     ) and (isinstance(raw_speech[0]			,  (np.ndarray, tuple, list)     ))
   )

   if is_batched:
    lowerCamelCase_      =  [np.asarray(UpperCamelCase			,  dtype=np.floataa     ) for speech in raw_speech]
   elif not is_batched and not isinstance(UpperCamelCase			,  np.ndarray     ):
    lowerCamelCase_      =  np.asarray(UpperCamelCase			,  dtype=np.floataa     )
   elif isinstance(UpperCamelCase			,  np.ndarray     ) and raw_speech.dtype is np.dtype(np.floataa     ):
    lowerCamelCase_      =  raw_speech.astype(np.floataa     )

   # always return batch
   if not is_batched:
    lowerCamelCase_      =  [raw_speech]

   # extract fbank features
   lowerCamelCase_      =  [self._extract_fbank_features(UpperCamelCase     ) for waveform in raw_speech]

   # convert into correct format for padding
   lowerCamelCase_      =  BatchFeature({"input_features": features}     )

   lowerCamelCase_      =  self.pad(
       UpperCamelCase			,  padding=UpperCamelCase			,  max_length=UpperCamelCase			,  truncation=UpperCamelCase			,  pad_to_multiple_of=UpperCamelCase			,  return_attention_mask=UpperCamelCase			,  **UpperCamelCase			,  )

   # make sure list is in array format
   lowerCamelCase_      =  padded_inputs.get("input_features"     )
   if isinstance(input_features[0]			,  UpperCamelCase     ):
    lowerCamelCase_      =  [np.asarray(UpperCamelCase			,  dtype=np.floataa     ) for feature in input_features]

   lowerCamelCase_      =  padded_inputs.get("attention_mask"     )
   if attention_mask is not None:
    lowerCamelCase_      =  [np.asarray(UpperCamelCase			,  dtype=np.intaa     ) for array in attention_mask]

   # Utterance-level cepstral mean and variance normalization
   if self.do_ceptral_normalize:
    lowerCamelCase_      =  (
        np.array(UpperCamelCase			,  dtype=np.intaa     )
        if self._get_padding_strategies(UpperCamelCase			,  max_length=UpperCamelCase     ) is not PaddingStrategy.DO_NOT_PAD
        else None
    )
    lowerCamelCase_      =  self.normalize(
        padded_inputs["input_features"]			,  attention_mask=UpperCamelCase     )

   if return_tensors is not None:
    lowerCamelCase_      =  padded_inputs.convert_to_tensors(UpperCamelCase     )

   return padded_inputs



