





import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


_UpperCAmelCase										=					datasets.utils.logging.get_logger(__name__)



@dataclass
class 				_UpperCamelCase       (						datasets.BuilderConfig  ):
       _UpperCamelCase					:       Optional[datasets.Features]       = None
       _UpperCamelCase					:       str       = "utf-8"
       _UpperCamelCase					:       Optional[str]       = None
       _UpperCamelCase					:       int       = 1_0 << 2_0  # 10MB
       _UpperCamelCase					:       bool       = False
       _UpperCamelCase					:       str       = "line"



class 				_UpperCamelCase       (						datasets.ArrowBasedBuilder  ):
       _UpperCamelCase					:       Dict       = TextConfig

       def lowercase			(		self:			Tuple		)							->			int:




              """simple docstring"""


              return datasets.DatasetInfo(features=self.config.features		)

       def lowercase			(		self:			Tuple       , _SCREAMING_SNAKE_CASE:			int		)							->			List[str]:




              """simple docstring"""


              if not self.config.data_files:
                     raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}'''		)
              UpperCamelCase_ 							=	dl_manager.download_and_extract(self.config.data_files		)
              if isinstance(UpperCAmelCase__       , (str, list, tuple)		):
                     UpperCamelCase_ 							=	data_files
                     if isinstance(UpperCAmelCase__       , UpperCAmelCase__		):
                            UpperCamelCase_ 							=	[files]
                     UpperCamelCase_ 							=	[dl_manager.iter_files(UpperCAmelCase__		) for file in files]
                     return [datasets.SplitGenerator(name=datasets.Split.TRAIN       , gen_kwargs={"files": files}		)]
              UpperCamelCase_ 							=	[]
              for split_name, files in data_files.items():
                     if isinstance(UpperCAmelCase__       , UpperCAmelCase__		):
                            UpperCamelCase_ 							=	[files]
                     UpperCamelCase_ 							=	[dl_manager.iter_files(UpperCAmelCase__		) for file in files]
                     splits.append(datasets.SplitGenerator(name=UpperCAmelCase__       , gen_kwargs={"files": files}		)		)
              return splits

       def lowercase			(		self:			Dict       , _SCREAMING_SNAKE_CASE:			pa.Table		)							->			pa.Table:




              """simple docstring"""


              if self.config.features is not None:
                     UpperCamelCase_ 							=	self.config.features.arrow_schema
                     if all(not require_storage_cast(UpperCAmelCase__		) for feature in self.config.features.values()		):
                            # cheaper cast
                            UpperCamelCase_ 							=	pa_table.cast(UpperCAmelCase__		)
                     else:
                            # more expensive cast; allows str <-> int/float or str to Audio for example
                            UpperCamelCase_ 							=	table_cast(UpperCAmelCase__       , UpperCAmelCase__		)
                     return pa_table
              else:
                     return pa_table.cast(pa.schema({"text": pa.string()}		)		)



       def lowercase			(		self:			Optional[int]       , _SCREAMING_SNAKE_CASE:			str		)							->			str:




              """simple docstring"""


              UpperCamelCase_ 							=	list(self.config.features		) if self.config.features is not None else ["""text"""]
              for file_idx, file in enumerate(itertools.chain.from_iterable(UpperCAmelCase__		)		):
                     # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
                     with open(UpperCAmelCase__       , encoding=self.config.encoding       , errors=self.config.errors		) as f:
                            if self.config.sample_by == "line":
                                   UpperCamelCase_ 							=	0
                                   while True:
                                          UpperCamelCase_ 							=	f.read(self.config.chunksize		)
                                          if not batch:
                                                 break
                                          batch += f.readline()  # finish current line
                                          # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
                                          UpperCamelCase_ 							=	StringIO(UpperCAmelCase__		).readlines()
                                          if not self.config.keep_linebreaks:
                                                 UpperCamelCase_ 							=	[line.rstrip("\n"		) for line in batch]
                                          UpperCamelCase_ 							=	pa.Table.from_arrays([pa.array(UpperCAmelCase__		)]       , names=UpperCAmelCase__		)
                                          # Uncomment for debugging (will print the Arrow table size and elements)
                                          # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                          # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                          yield (file_idx, batch_idx), self._cast_table(UpperCAmelCase__		)
                                          batch_idx += 1
                            elif self.config.sample_by == "paragraph":
                                   UpperCamelCase_ 							=	0
                                   UpperCamelCase_ 							=	""""""
                                   while True:
                                          UpperCamelCase_ 							=	f.read(self.config.chunksize		)
                                          if not new_batch:
                                                 break
                                          batch += new_batch
                                          batch += f.readline()  # finish current line
                                          UpperCamelCase_ 							=	batch.split("\n\n"		)
                                          UpperCamelCase_ 							=	pa.Table.from_arrays(
                                              [pa.array([example for example in batch[:-1] if example]		)]       , names=UpperCAmelCase__		)
                                          # Uncomment for debugging (will print the Arrow table size and elements)
                                          # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                          # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                          yield (file_idx, batch_idx), self._cast_table(UpperCAmelCase__		)
                                          batch_idx += 1
                                          UpperCamelCase_ 							=	batch[-1]
                                   if batch:
                                          UpperCamelCase_ 							=	pa.Table.from_arrays([pa.array([batch]		)]       , names=UpperCAmelCase__		)
                                          yield (file_idx, batch_idx), self._cast_table(UpperCAmelCase__		)
                            elif self.config.sample_by == "document":
                                   UpperCamelCase_ 							=	f.read()
                                   UpperCamelCase_ 							=	pa.Table.from_arrays([pa.array([text]		)]       , names=UpperCAmelCase__		)
                                   yield file_idx, self._cast_table(UpperCAmelCase__		)




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_speech_available,
    is_torch_available,
)


UpperCAmelCase__			:  Tuple					     =     {
    """configuration_speech_to_text_2""": ["""SPEECH_TO_TEXT_2_PRETRAINED_CONFIG_ARCHIVE_MAP""", """Speech2Text2Config"""],
    """processing_speech_to_text_2""": ["""Speech2Text2Processor"""],
    """tokenization_speech_to_text_2""": ["""Speech2Text2Tokenizer"""],
}


try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase__			:  Tuple					     =     [
      """SPEECH_TO_TEXT_2_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """Speech2Text2ForCausalLM""",
      """Speech2Text2PreTrainedModel""",
  ]


if TYPE_CHECKING:
  from .configuration_speech_to_text_a import SPEECH_TO_TEXT_2_PRETRAINED_CONFIG_ARCHIVE_MAP, SpeechaTextaConfig
  from .processing_speech_to_text_a import SpeechaTextaProcessor
  from .tokenization_speech_to_text_a import SpeechaTextaTokenizer

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_speech_to_text_a import (
        SPEECH_TO_TEXT_2_PRETRAINED_MODEL_ARCHIVE_LIST,
        SpeechaTextaForCausalLM,
        SpeechaTextaPreTrainedModel,
    )

else:
  import sys

  UpperCAmelCase__			:  Optional[int]					     =     _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




